Run Chrome Endure tests with network simulation via Web Page Replay.

chrome/test/functional/perf.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Basic pyauto performance tests.
 
 For tests that need to be run for multiple iterations (e.g., so that average
 and standard deviation values can be reported), the default number of iterations
 run for each of these tests is specified by |_DEFAULT_NUM_ITERATIONS|.
@@ skipping 1710 matching lines @@
         fps = result.repeat_frame_times.GetFps()
         assert fps, '%s did not scroll' % url
         logging.info('Iteration %d of %d: %f fps', iteration,
                      self._num_iterations, fps)
         results.append(result)
     self._PrintSummaryResults(
         description, [r.repeat_frame_times.GetFps() for r in results],
         'FPS', graph_name)
 
 
+class BaseWebPageReplay(object):
+  """Run tests with network simulation via Web Page Replay.
+
+  Web Page Replay is a proxy that can record and "replay" web pages with
+  simulated network characteristics -- without having to edit the pages
+  by hand. With WPR, tests can use "real" web content, and catch
+  performance issues that may result from introducing network delays and
+  bandwidth throttling.
+
+  This class provides a minimum set of settings for running tests with
+  Web Page Replay.
+
+  Environment Variables:
+    WPR_RECORD: if set, puts Web Page Replay in record mode instead of replay.
+    WPR_REPLAY_DIR: path to alternate Web Page Replay source (for development).
+    WPR_ARCHIVE_PATH: path to alternate archive file (e.g. '/tmp/foo.wpr').
+  """
+  _BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                           '..', '..', '..', '..'))
+
+  def Path(self, key, **kwargs):
+    """Provide paths for tests using Web Page Replay."""
+    chromium_path = self._paths[key].format(**kwargs)
+    return os.path.join(self._BASE_DIR, *chromium_path.split('/'))
+
+  def _ArchivePath(self, test_name):
+    archive_path = self.archive_path or self.Path('archive',
+                                                  test_name=test_name)
+

[slamm_google, 2012/07/25 17:20:48]

Remove extra blank line.

[fdeng1, 2012/07/25 22:53:28]

Done.

+    if self.is_record_mode:
+      archive_dir = os.path.dirname(archive_path)
+      assert os.path.exists(archive_dir), \
+          'Archive directory does not exist: %s' % archive_dir
+    else:
+      assert os.path.exists(archive_path), \
+          'Archive file path does not exist: %s' % archive_path
+    return archive_path
+
+  def __init__(self):

[slamm_google, 2012/07/25 17:20:48]

Move __init__ before Path().

[fdeng1, 2012/07/25 22:53:28]

Done.

+    # Initialize Web Page Replay related paths

[dennis_jeffrey, 2012/07/24 21:03:19]

add period at end of sentence

[slamm_google, 2012/07/25 17:20:48]

How about dropping the comment? And the others too. They do not add extra value.

[fdeng1, 2012/07/25 22:53:28]

Done.

+    self._paths = {
+        'archive': os.path.join(os.getcwd(), '{test_name}.wpr'),
+        'replay': 'src/third_party/webpagereplay',
+        'logs': 'src/webpagereplay_logs',
+        }
+
+    # Initialize extra flags for chrome.
+    self.chrome_flags = [
+        ('--host-resolver-rules=MAP * %s,EXCLUDE localhost' %
+         webpagereplay.REPLAY_HOST),
+        '--testing-fixed-http-port=%s' % webpagereplay.HTTP_PORT,
+        '--testing-fixed-https-port=%s' % webpagereplay.HTTPS_PORT,
+        '--ignore-certificate-errors',
+        ]
+
+    # Initialize Web Page Replay flags.
+    self._replay_flags = ['--no-dns_forwarding']
+
+    self.archive_path = os.environ.get('WPR_ARCHIVE_PATH')
+    self.replay_dir = os.environ.get('WPR_REPLAY_DIR', self.Path('replay'))
+    self.is_record_mode = 'WPR_RECORD' in os.environ
+    if self.is_record_mode:
+      self._num_iterations = 1

[slamm_google, 2012/07/25 17:20:48]

This is a bug from an earlier change. Need to find the right place to set this.

[fdeng1, 2012/07/25 22:53:28]

I moved it the setSep() functions in PageCyclerNetSimTest and
PopularSitesScrollTest. How do you think?

On 2012/07/25 17:20:48, slamm wrote:

+
+  def GetReplayServer(self, test_name):
+    """Create a replay server.
+
+    Args:
+      test_name: the name of the test which is starting the server.
+
+    Returns:
+      A replay server.
+    """
+    replay_flags = self._replay_flags
+    if self.is_record_mode:
+      replay_flags += ['--record']
+
+    return webpagereplay.ReplayServer(
+        self.replay_dir,
+        self._ArchivePath(test_name),
+        self.Path('logs'),
+        replay_flags)
+
+
+class PerfWebPageReplay(BaseWebPageReplay):
+  """Run page cycler and scroll tests with network simulation.
+
+  This class allows customized settings for page cycler and scroll tests.
+  """
+
+  def __init__(self):
+    # Call parent __init__

[slamm_google, 2012/07/25 17:20:48]

The comments in this method do not add any value and can be dropped.

[fdeng1, 2012/07/25 22:53:28]

Done.

+    super(PerfWebPageReplay, self).__init__()
+
+   # Extra paths for Web Page Replay
+    extra_paths = {
+        'archive': 'src/data/page_cycler/webpagereplay/{test_name}.wpr',
+        'page_sets': 'src/tools/page_cycler/webpagereplay/tests/{test_name}.js',
+        'start_page': 'src/tools/page_cycler/webpagereplay/start.html',
+        'extension': 'src/tools/page_cycler/webpagereplay/extension',
+        }
+    self._paths.update(extra_paths)
+
+    # Extra chrome flags
+    extra_chrome_flags = [
+        '--log-level=0',
+        '--disable-background-networking',
+        '--enable-experimental-extension-apis',
+        ' --enable-logging',

[slamm_google, 2012/07/25 17:20:48]

Remove the extra whitespace from the flag.

[fdeng1, 2012/07/25 22:53:28]

Done.

+        '--enable-stats-table',
+        '--enable-benchmarking',
+        '--metrics-recording-only',
+        '--activate-on-launch',
+        ]
+    self.chrome_flags.extend(extra_chrome_flags)
+
+
 class PopularSitesScrollTest(BaseScrollTest):
   """Measures scrolling performance on recorded versions of popular sites."""
 
+  _web_page_replay = PerfWebPageReplay()
+
   def ExtraChromeFlags(self):
     """Ensures Chrome is launched with custom flags.
 
     Returns:
       A list of extra flags to pass to Chrome when it is launched.
     """
     return super(PopularSitesScrollTest,
-                 self).ExtraChromeFlags() + WebPageReplay.CHROME_FLAGS
+                 self).ExtraChromeFlags() + self._web_page_replay.chrome_flags
 
   def _GetUrlList(self, test_name):
     """Returns list of recorded sites."""
-    with open(WebPageReplay.Path('page_sets', test_name=test_name)) as f:
+    with open(
+        self._web_page_replay.Path('page_sets', test_name=test_name)) as f:
       sites_text = f.read()
     js = """
       %s
       window.domAutomationController.send(JSON.stringify(pageSets));
     """ % sites_text
     page_sets = eval(self.ExecuteJavascript(js))
     return list(itertools.chain(*page_sets))[1:]  # Skip first.
 
   def _PrintScrollResults(self, results):
     self._PrintSummaryResults(
@@ skipping 11 matching lines @@
         [r.repeat_frame_times.GetPercentBelow60Fps() for r in results],
         'percent', 'PercentBelow60FPS')
     self._PrintSummaryResults(
         'first_paint_time', [r.first_paint_time for r in results],
         'ms', 'FirstPaintTime')
 
   def test2012Q3(self):
     test_name = '2012Q3'
     urls = self._GetUrlList(test_name)
     results = []
-    with WebPageReplay().GetReplayServer(test_name):
+    with self._web_page_replay.GetReplayServer(test_name):
       for iteration in range(self._num_iterations):
         for url in urls:
           result = self.RunSingleInvocation(url)
           fps = result.initial_frame_times.GetFps()
           assert fps, '%s did not scroll' % url
           logging.info('Iteration %d of %d: %f fps', iteration,
                        self._num_iterations, fps)
           results.append(result)
     self._PrintScrollResults(results)
 
@@ skipping 380 matching lines @@
   def testIntl2File(self):
     self.RunPageCyclerTest('intl2', 'Intl2File')
 
   def testMozFile(self):
     self.RunPageCyclerTest('moz', 'MozFile')
 
   def testMoz2File(self):
     self.RunPageCyclerTest('moz2', 'Moz2File')
 
 
-class WebPageReplay(object):
-  """Run page cycler tests with network simulation via Web Page Replay.
-
-  Web Page Replay is a proxy that can record and "replay" web pages with
-  simulated network characteristics -- without having to edit the pages
-  by hand. With WPR, tests can use "real" web content, and catch
-  performance issues that may result from introducing network delays and
-  bandwidth throttling.
-
-  Environment Variables:
-    WPR_RECORD: if set, puts Web Page Replay in record mode instead of replay.
-    WPR_REPLAY_DIR: path to alternate Web Page Replay source (for development).
-    WPR_ARCHIVE_PATH: path to alternate archive file (e.g. '/tmp/foo.wpr').
-  """
-  _PATHS = {
-      'archive':    'src/data/page_cycler/webpagereplay/{test_name}.wpr',
-      'page_sets':  'src/tools/page_cycler/webpagereplay/tests/{test_name}.js',
-      'start_page': 'src/tools/page_cycler/webpagereplay/start.html',
-      'extension':  'src/tools/page_cycler/webpagereplay/extension',
-      'replay':     'src/third_party/webpagereplay',
-      'logs':       'src/webpagereplay_logs',
-      }
-
-  _BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                           '..', '..', '..', '..'))
-  CHROME_FLAGS = [
-      '--host-resolver-rules=MAP * %s' % webpagereplay.REPLAY_HOST,
-      '--testing-fixed-http-port=%s' % webpagereplay.HTTP_PORT,
-      '--testing-fixed-https-port=%s' % webpagereplay.HTTPS_PORT,
-      '--log-level=0',
-      '--disable-background-networking',
-      '--enable-experimental-extension-apis',
-      '--enable-logging',
-      '--enable-stats-table',
-      '--enable-benchmarking',
-      '--ignore-certificate-errors',
-      '--metrics-recording-only',
-      '--activate-on-launch',
-      '--no-first-run',
-      '--no-proxy-server',
-      ]
-
-  @classmethod
-  def Path(cls, key, **kwargs):
-    """Provide paths for tests using Web Page Replay."""
-    chromium_path = cls._PATHS[key].format(**kwargs)
-    return os.path.join(cls._BASE_DIR, *chromium_path.split('/'))
-
-  def _ArchivePath(self, test_name):
-    archive_path = self.archive_path or self.Path('archive',
-                                                  test_name=test_name)
-    if self.is_record_mode:
-      archive_dir = os.path.dirname(archive_path)
-      assert os.path.exists(archive_dir), \
-          'Archive directory does not exist: %s' % archive_dir
-    else:
-      assert os.path.exists(archive_path), \
-          'Archive file path does not exist: %s' % archive_path
-    return archive_path
-
-  def __init__(self):
-    self.archive_path = os.environ.get('WPR_ARCHIVE_PATH')
-    self.replay_dir = os.environ.get('WPR_REPLAY_DIR', self.Path('replay'))
-    self.is_record_mode = 'WPR_RECORD' in os.environ
-    if self.is_record_mode:
-      self._num_iterations = 1
-
-  def GetReplayServer(self, test_name):
-    replay_options = []
-    replay_options.append('--no-dns_forwarding')
-    if self.is_record_mode:
-      replay_options.append('--record')
-    return webpagereplay.ReplayServer(
-        self.replay_dir,
-        self._ArchivePath(test_name),
-        self.Path('logs'),
-        replay_options)
-
-
 class PageCyclerNetSimTest(BasePageCyclerTest):
   """Tests to run Web Page Replay backed page cycler tests."""
   MAX_ITERATION_SECONDS = 180
+  _web_page_replay = PerfWebPageReplay()
 
   def ExtraChromeFlags(self):
     """Ensures Chrome is launched with custom flags.
 
     Returns:
       A list of extra flags to pass to Chrome when it is launched.
     """
     flags = super(PageCyclerNetSimTest, self).ExtraChromeFlags()
-    flags.append('--load-extension=%s' % WebPageReplay.Path('extension'))
-    flags.extend(WebPageReplay.CHROME_FLAGS)
+    flags.append('--load-extension=%s' %
+                 self._web_page_replay.Path('extension'))
+    flags.extend(self._web_page_replay.chrome_flags)
     return flags
 
   def StartUrl(self, test_name, iterations):
     start_url = 'file://%s?test=%s&iterations=%d' % (
-        WebPageReplay.Path('start_page'), test_name, iterations)
+        self._web_page_replay.Path('start_page'), test_name, iterations)
     if self.use_auto:
       start_url += '&auto=1'
     return start_url
 
   def RunPageCyclerTest(self, test_name, description):
     """Runs the specified PageCycler test.
 
     Args:
       test_name: name for archive (.wpr) and config (.js) files.
       description: a string description for the test
     """
-    with WebPageReplay().GetReplayServer(test_name):
+    with self._web_page_replay.GetReplayServer(test_name):
       super_self = super(PageCyclerNetSimTest, self)
       super_self.RunPageCyclerTest(test_name, description)
 
   def test2012Q2(self):
     self.RunPageCyclerTest('2012Q2', '2012Q2')
 
 
 class MemoryTest(BasePerfTest):
   """Tests to measure memory consumption under different usage scenarios."""
 
@@ skipping 327 matching lines @@
     """Identifies the port number to which the server is currently bound.
 
     Returns:
       The numeric port number to which the server is currently bound.
     """
     return self._server.server_address[1]
 
 
 if __name__ == '__main__':
   pyauto_functional.Main()