Run Chrome Endure tests with network simulation via Web Page Replay.

chrome/test/functional/perf_endure.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance tests for Chrome Endure (long-running perf tests on Chrome).
 
 This module accepts the following environment variable inputs:
   TEST_LENGTH: The number of seconds in which to run each test.
   PERF_STATS_INTERVAL: The number of seconds to wait in-between each sampling
       of performance/memory statistics.
 
   DEEP_MEMORY_PROFILE: Enable the Deep Memory Profiler if it's set to 'True'.
   DEEP_MEMORY_PROFILE_INTERVAL: The number of seconds to wait in-between each
       sampling for the Deep Memory Profiler.
   DEEP_MEMORY_PROFILE_SAVE: Don't clean up dump files if it's set to 'True'.
+
+  ENDURE_NO_WPR: Run tests without Web Page Replay if it's set.
+  WPR_RECORD: Run tests in record mode. If you want to make a fresh
+              archive, please make sure to delete the old one, otherwise
+              it will append to the old one.

[Nirnimesh, 2012/08/07 17:17:34]

remove 'please'

[fdeng, 2012/08/07 21:24:19]

Done.

+  WPR_ARCHIVE_PATH: an alternative archive file to use.
 """
 
 from datetime import datetime
 import json
 import logging
 import os
 import re
 import subprocess
 import tempfile
 import time
 
 import perf
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 import pyauto_errors
 import pyauto_utils
 import remote_inspector_client
 import selenium.common.exceptions
 from selenium.webdriver.support.ui import WebDriverWait
+import webpagereplay
 
 
 class NotSupportedEnvironmentError(RuntimeError):
   """Represent an error raised since the environment (OS) is not supported."""
   pass
 
+
 class ChromeEndureBaseTest(perf.BasePerfTest):
   """Implements common functionality for all Chrome Endure tests.
 
   All Chrome Endure test classes should inherit from this class.
   """
 
   _DEFAULT_TEST_LENGTH_SEC = 60 * 60 * 6  # Tests run for 6 hours.
   _GET_PERF_STATS_INTERVAL = 60 * 5  # Measure perf stats every 5 minutes.
   # TODO(dennisjeffrey): Do we still need to tolerate errors?
   _ERROR_COUNT_THRESHOLD = 50  # Number of ChromeDriver errors to tolerate.
   _DEEP_MEMORY_PROFILE = False
   _DEEP_MEMORY_PROFILE_INTERVAL = _GET_PERF_STATS_INTERVAL
   _DEEP_MEMORY_PROFILE_SAVE = False
 
   _DMPROF_DIR_PATH = os.path.join(
       os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
       'tools', 'deep_memory_profiler')
 
   _DMPROF_SCRIPT_PATH = os.path.join(_DMPROF_DIR_PATH, 'dmprof')
 
   _CHROME_BIN_PATH = os.path.join(perf.BasePerfTest.BrowserPath(), 'chrome')
 
   def setUp(self):
+    # Parse the environment variable for the usage of Web Page Replay.
+    # It must be parsed before perf.BasePerfTest.setUp()

[slamm_google, 2012/08/07 20:00:43]

Better?

# The Web Page Replay environment variables must be parsed before
# perf.BasePerfTest.setUp().

[fdeng, 2012/08/07 21:24:19]

Done.

+    self._ParseReplayEnv()
     # The environment variables for the Deep Memory Profiler must be set
     # before perf.BasePerfTest.setUp() to inherit them to Chrome.
     self._deep_memory_profile = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE', bool, self._DEEP_MEMORY_PROFILE)
 
     self._deep_memory_profile_interval = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE_INTERVAL', int, self._DEEP_MEMORY_PROFILE_INTERVAL)
 
     if self._deep_memory_profile:
       if not self.IsLinux():
@@ skipping 35 matching lines @@
     self._remote_inspector_client = (
         remote_inspector_client.RemoteInspectorClient())
     logging.info('Connection to remote inspector set up successfully.')
 
     self._test_start_time = 0
     self._num_errors = 0
     self._events_to_output = []
     self._deep_memory_profile_json_file = None
     self._deep_memory_profile_last_json_filename = ''
     self._deep_memory_profile_proc = None
+    self._StartReplayServerIfNecessary()
 
   def tearDown(self):
     logging.info('Terminating connection to remote inspector...')
     self._remote_inspector_client.Stop()
     logging.info('Connection to remote inspector terminated.')
     if self._deep_memory_profile:
       # TODO(dmikurube): Stop to set HEAP_PROFILE_TIME_INTERVAL in setUp when
       # PyAuto supports to dump renderer heap profile.
       del os.environ['HEAP_PROFILE_TIME_INTERVAL']
       del os.environ['DEEP_HEAP_PROFILE']
       del os.environ['HEAP_PROFILE_MMAP']
       del os.environ['HEAPPROFILE']
 
     # Must be done at end of this function except for post-cleaning after
     # Chrome finishes.
     perf.BasePerfTest.tearDown(self)
 
     # Remove the temporary directory after Chrome finishes in tearDown.
     if (self._deep_memory_profile and
         not self._deep_memory_profile_save and
         self._deep_tempdir):
       pyauto_utils.RemovePath(self._deep_tempdir)
+    # Must be done after perf.BasePerfTest.tearDown()
+    self._StopReplayServerIfNecessary()
+
+  def _GetArchiveName(self):
+    """Return the Web Page Replay archive name that corresponds to a test.
+
+    Override this function to return the name of an archive that
+    corresponds to the test, e.g "ChromeEndureGmailTest.wpr".
+
+    Returns:
+      None, by default no archive name is provided.
+    """
+    return None
+
+  def _ParseReplayEnv(self):
+    """Parse Web Page Replay related envrionment variables."""
+    if 'ENDURE_NO_WPR' in os.environ:
+      self._use_wpr = False
+      logging.info('Skipping Web Page Replay since ENDURE_NO_WPR is set.')
+    else:
+      self._archive_path = None
+      if 'WPR_ARCHIVE_PATH' in os.environ:
+        self._archive_path = os.environ.get('WPR_ARCHIVE_PATH')
+      else:
+        if self._GetArchiveName():
+          self._archive_path = ChromeEndureReplay.Path(
+              'archive', archive_name=self._GetArchiveName())
+      self._is_record_mode = 'WPR_RECORD' in os.environ
+      if self._is_record_mode:
+        if self._archive_path:
+          self._use_wpr = True

[fdeng1, 2012/08/07 02:56:57]

In record mode, we just need an valid archive file path. It is ok if the archive
doesn't exist.

+        else:
+          self._use_wpr = False
+          logging.info('Fail to record since a valid archive path can not ' +
+                       'be generated. Did you implement ' +
+                       '_GetArchiveName() in your test?')
+      else:
+        if self._archive_path and os.path.exists(self._archive_path):
+          self._use_wpr = True

[fdeng1, 2012/08/07 02:56:57]

In replay mode, we need an valid archive file path and the file must exist.

+        else:
+          self._use_wpr = False
+          logging.info(
+              'Skipping Web Page Replay since archive file %sdoes not exist.',
+              self._archive_path+' ' if self._archive_path else '')

[dennis_jeffrey, 2012/08/07 19:42:09]

add a space right before and right after the +

[fdeng, 2012/08/07 21:24:19]

Done.

 
   def _GetDeepMemoryProfileEnv(self, env_name, converter, default):
     """Returns a converted environment variable for the Deep Memory Profiler.
 
     Args:
       env_name: A string name of an environment variable.
       converter: A function taking a string to convert an environment variable.
       default: A value used if the environment variable is not specified.
 
     Returns:
@@ skipping 21 matching lines @@
     # The same with setUp, but need to fetch the environment variable since
     # ExtraChromeFlags is called before setUp.
     deep_memory_profile = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE', bool, self._DEEP_MEMORY_PROFILE)
 
     # Ensure Chrome enables remote debugging on port 9222.  This is required to
     # interact with Chrome's remote inspector.
     extra_flags = ['--remote-debugging-port=9222']
     if deep_memory_profile:
       extra_flags.append('--no-sandbox')
-    return (perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags)
+    if self._use_wpr:
+      extra_flags.extend(ChromeEndureReplay.CHROME_FLAGS)
+    return perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags
 
   def _OnTimelineEvent(self, event_info):
     """Invoked by the Remote Inspector Client when a timeline event occurs.
 
     Args:
       event_info: A dictionary containing raw information associated with a
          timeline event received from Chrome's remote inspector.  Refer to
          chrome/src/third_party/WebKit/Source/WebCore/inspector/Inspector.json
          for the format of this dictionary.
     """
@@ skipping 339 matching lines @@
     try:
       element = self._GetElement(driver.find_element_by_xpath, xpath)
       element.click()
     except (selenium.common.exceptions.StaleElementReferenceException,
             selenium.common.exceptions.TimeoutException) as e:
       logging.exception('WebDriver exception: %s' % e)
       return False
 
     return True
 
+  def _StartReplayServerIfNecessary(self):
+    """Start replay server if necessary.
+
+    This method needs to be called BEFORE any connection (which is supposed
+    to go through the Web Page Replay server) occurs.

[slamm_google, 2012/08/07 20:00:43]

Does this need to be said?

[fdeng, 2012/08/07 21:24:19]

Deleted.
On 2012/08/07 20:00:43, slamm wrote:

+    """
+    if self._use_wpr:
+      mode = 'record' if self._is_record_mode else 'replay'
+      self._wpr_server = ChromeEndureReplay.ReplayServer(self._archive_path)
+      self._wpr_server.StartServer()
+      logging.info('Web Page Replay server has started in %s mode.', mode)
+
+  def _StopReplayServerIfNecessary(self):
+    """Stop the Web Page Replay server if necessary.
+
+    This method has to be called AFTER all network connections which go
+    through Web Page Replay server have shut down. Otherwise the
+    Web Page Replay server will hang to wait for them. A good
+    place is to call it at the end of the teardown process.
+    """
+    if self._use_wpr:
+      self._wpr_server.StopServer()
+      logging.info('The Web Page Replay server stopped.')
+
 
 class ChromeEndureControlTest(ChromeEndureBaseTest):
   """Control tests for Chrome Endure."""
 
   _WEBAPP_NAME = 'Control'
   _TAB_TITLE_SUBSTRING = 'Chrome Endure Control Test'
 
   def testControlAttachDetachDOMTree(self):
     """Continually attach and detach a DOM tree from a basic document."""
     test_description = 'AttachDetachDOMTree'
@@ skipping 63 matching lines @@
 
     # Wait until Gmail's 'canvas_frame' loads and the 'Inbox' link is present.
     # TODO(dennisjeffrey): Check with the Gmail team to see if there's a better
     # way to tell when the webpage is ready for user interaction.
     self._wait.until(
         self._SwitchToCanvasFrame)  # Raises exception if the timeout is hit.
     # Wait for the inbox to appear.
     self.WaitForDomNode('//a[starts-with(@title, "Inbox")]',
                         frame_xpath=self._FRAME_XPATH)
 
+  def _GetArchiveName(self):
+    """Return Web Page Replay archive name."""
+    return 'ChromeEndureGmailTest.wpr'
+
   def _SwitchToCanvasFrame(self, driver):
     """Switch the WebDriver to Gmail's 'canvas_frame', if it's available.
 
     Args:
       driver: A selenium.webdriver.remote.webdriver.WebDriver object.
 
     Returns:
       True, if the switch to Gmail's 'canvas_frame' is successful, or
       False if not.
     """
@@ skipping 307 matching lines @@
     self.NavigateToURL('http://docs.google.com')
     self.assertTrue(
         self.WaitUntil(lambda: self._TAB_TITLE_SUBSTRING in
                                self.GetActiveTabTitle(),
                        timeout=60, expect_retval=True, retry_sleep=1),
         msg='Timed out waiting for Docs to load. Tab title is: %s' %
             self.GetActiveTabTitle())
 
     self._driver = self.NewWebDriver()
 
+  def _GetArchiveName(self):
+    """Return Web Page Replay archive name."""
+    return 'ChromeEndureDocsTest.wpr'
+
   def testDocsAlternatelyClickLists(self):
     """Alternates between two different document lists.
 
     This test alternately clicks the "Shared with me" and "My Drive" buttons in
     Google Docs, and periodically gathers performance stats that may reveal
     memory bloat.
     """
     test_description = 'AlternateLists'
 
     def sort_menu_setup():
@@ skipping 50 matching lines @@
     # Log into a test Google account and open up Google Plus.
     self._LoginToGoogleAccount()
     self.NavigateToURL('http://plus.google.com')
     loaded_tab_title = self.GetActiveTabTitle()
     self.assertTrue(self._TAB_TITLE_SUBSTRING in loaded_tab_title,
                     msg='Loaded tab title does not contain "%s": "%s"' %
                         (self._TAB_TITLE_SUBSTRING, loaded_tab_title))
 
     self._driver = self.NewWebDriver()
 
+  def _GetArchiveName(self):
+    """Return Web Page Replay archive name."""
+    return 'ChromeEndurePlusTest.wpr'
+
   def testPlusAlternatelyClickStreams(self):
     """Alternates between two different streams.
 
     This test alternately clicks the "Friends" and "Family" buttons using
     Google Plus, and periodically gathers performance stats that may reveal
     memory bloat.
     """
     test_description = 'AlternateStreams'
 
     def scenario():
@@ skipping 83 matching lines @@
       except (pyauto_errors.JSONInterfaceError,
               pyauto_errors.JavascriptRuntimeError):
         self._num_errors += 1
 
       time.sleep(1)
 
     self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING,
                         test_description, scenario)
 
 
+class ChromeEndureReplay(object):
+  """Run Chrome Endure tests with network simulation via Web Page Replay."""
+
+  _PATHS = {
+      'archive':
+      'src/chrome/test/data/pyauto_private/webpagereplay/{archive_name}',
+      'scripts':
+      'src/chrome/test/data/chrome_endure/webpagereplay/wpr_deterministic.js',
+      }
+  CHROME_FLAGS = webpagereplay.CHROME_FLAGS
+
+  @classmethod
+  def Path(cls, key, **kwargs):
+    return perf.FormatChromePath(cls._PATHS[key], **kwargs)
+
+  @classmethod
+  def ReplayServer(cls, archive_path):
+    """Creat a replay server."""

[slamm_google, 2012/08/07 20:00:43]

Creat -> Create

[fdeng, 2012/08/07 21:24:19]

Done.

+    # Inject customized scripts for Google webapps.
+    # See the javascript file for details.
+    scripts = cls.Path('scripts')
+    if not os.path.exists(scripts):
+      raise webpagereplay.ReplayNotFoundError('injected scripts', scripts)

[slamm_google, 2012/08/07 20:00:43]

Instead of using a custom exception from webpagereplay, how about either
creating an exception for this module, or raise something like IOError?

[fdeng, 2012/08/07 21:24:19]

Change to IOError
On 2012/08/07 20:00:43, slamm wrote:

+    replay_options = ['--inject_scripts', scripts]
+    if 'WPR_RECORD' in os.environ:
+      replay_options.append('--append')
+    return webpagereplay.ReplayServer(archive_path, replay_options)
+
+
 if __name__ == '__main__':
   pyauto_functional.Main()