Run Chrome Endure tests with network simulation via Web Page Replay.

chrome/test/functional/perf_endure.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance tests for Chrome Endure (long-running perf tests on Chrome).
 
 This module accepts the following environment variable inputs:
   TEST_LENGTH: The number of seconds in which to run each test.
   PERF_STATS_INTERVAL: The number of seconds to wait in-between each sampling
@@ skipping 15 matching lines @@
 import time
 
 import perf
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 import pyauto_errors
 import pyauto_utils
 import remote_inspector_client
 import selenium.common.exceptions
 from selenium.webdriver.support.ui import WebDriverWait
+import webpagereplay
 
 
 class NotSupportedEnvironmentError(RuntimeError):
   """Represent an error raised since the environment (OS) is not supported."""
   pass
 
+
 class ChromeEndureBaseTest(perf.BasePerfTest):
   """Implements common functionality for all Chrome Endure tests.
 
   All Chrome Endure test classes should inherit from this class.
   """
 
   _DEFAULT_TEST_LENGTH_SEC = 60 * 60 * 6  # Tests run for 6 hours.
   _GET_PERF_STATS_INTERVAL = 60 * 5  # Measure perf stats every 5 minutes.
   # TODO(dennisjeffrey): Do we still need to tolerate errors?
   _ERROR_COUNT_THRESHOLD = 50  # Number of ChromeDriver errors to tolerate.
   _DEEP_MEMORY_PROFILE = False
   _DEEP_MEMORY_PROFILE_INTERVAL = _GET_PERF_STATS_INTERVAL
   _DEEP_MEMORY_PROFILE_SAVE = False
 
   _DMPROF_DIR_PATH = os.path.join(
       os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
       'tools', 'deep_memory_profiler')
 
   _DMPROF_SCRIPT_PATH = os.path.join(_DMPROF_DIR_PATH, 'dmprof')
 
   _CHROME_BIN_PATH = os.path.join(perf.BasePerfTest.BrowserPath(), 'chrome')
 
   def setUp(self):
+    # Parse the environment variable for the usage of Web Page Replay.

[Nirnimesh, 2012/08/06 21:36:45]

please move the env parsing to a helper method so as not to litter this class
with wpr stuff

[fdeng1, 2012/08/07 02:56:56]

Done. move it to _ParseReplayEnv()

+    # It must be parsed before perf.BasePerfTest.setUp()
+    if 'ENDURE_NO_WPR' in os.environ:
+      self._use_wpr = False
+      logging.info('Web Page Replay is turned off as ENDURE_NO_WPR is set.')

[dennis_jeffrey, 2012/08/06 21:33:46]

how about this to match slamm's recommended comment below:

"Skipping Web Page Replay since ENDURE_NO_WPR is set".

[fdeng1, 2012/08/07 02:56:56]

Done.

+    else:
+      # Do NOT manually set WPR_RECORD for the purpose of
+      # making recording archive for endure tests. Run record_endure.py instead.
+      self._is_record_mode = 'WPR_RECORD' in os.environ
+      archive_path = os.environ.get('WPR_ARCHIVE_PATH',
+          ChromeEndureReplay.Path(
+              'archive', archive_name=self._GenArchiveName()))
+      if self._is_record_mode or os.path.exists(archive_path):
+        self._use_wpr = True

[dennis_jeffrey, 2012/08/06 21:33:46]

log a message in this case saying something like "Using Web Page Replay".

[fdeng1, 2012/08/07 02:56:56]

When WPR is successfully launched, _StartReplayServerIfNecessary will call
"logging.info('Web Page Replay server has started in %s mode.', mode)".
Do you think that is sufficient or maybe one more log here is better?

On 2012/08/06 21:33:46, dennis_jeffrey wrote:

+      else:
+        self._use_wpr = False
+        logging.info(
+            'Web Page Replay is turned off as the' +
+            ' archive file %s does not exist.',

[slamm_google, 2012/08/06 20:58:44]

How about the following?

Skipping Web Page Replay since the archive file, %s, does not exist.

[fdeng1, 2012/08/07 02:56:56]

Done.

+            self._GenArchiveName())
     # The environment variables for the Deep Memory Profiler must be set
     # before perf.BasePerfTest.setUp() to inherit them to Chrome.
     self._deep_memory_profile = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE', bool, self._DEEP_MEMORY_PROFILE)
 
     self._deep_memory_profile_interval = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE_INTERVAL', int, self._DEEP_MEMORY_PROFILE_INTERVAL)
 
     if self._deep_memory_profile:
       if not self.IsLinux():
@@ skipping 35 matching lines @@
     self._remote_inspector_client = (
         remote_inspector_client.RemoteInspectorClient())
     logging.info('Connection to remote inspector set up successfully.')
 
     self._test_start_time = 0
     self._num_errors = 0
     self._events_to_output = []
     self._deep_memory_profile_json_file = None
     self._deep_memory_profile_last_json_filename = ''
     self._deep_memory_profile_proc = None
+    self._StartReplayServerIfNecessary()
 
   def tearDown(self):
     logging.info('Terminating connection to remote inspector...')
     self._remote_inspector_client.Stop()
     logging.info('Connection to remote inspector terminated.')
     if self._deep_memory_profile:
       # TODO(dmikurube): Stop to set HEAP_PROFILE_TIME_INTERVAL in setUp when
       # PyAuto supports to dump renderer heap profile.
       del os.environ['HEAP_PROFILE_TIME_INTERVAL']
       del os.environ['DEEP_HEAP_PROFILE']
       del os.environ['HEAP_PROFILE_MMAP']
       del os.environ['HEAPPROFILE']
 
     # Must be done at end of this function except for post-cleaning after
     # Chrome finishes.
     perf.BasePerfTest.tearDown(self)
 
     # Remove the temporary directory after Chrome finishes in tearDown.
     if (self._deep_memory_profile and
         not self._deep_memory_profile_save and
         self._deep_tempdir):
       pyauto_utils.RemovePath(self._deep_tempdir)
+    # Must be done after perf.BasePerfTest.tearDown()
+    self._StopReplayServerIfNecessary()
+
+  def _GenArchiveName(self):
+    """Returns the Web Page Replay archive name that corresponds to a test.
+
+    Use test class name as archive name, e.g. ChromeEndureGmailTest.wpr,

[Nirnimesh, 2012/08/06 21:36:45]

While this is nifty to auto-generate the archive name, it hampers
discoverability.
Say I'm a random developer changing X.wpr and i want to find out all the tests
that use it. I'll not find this file in code search.

It's fine to just set a convention that the archive file matches the class name.
It's not necessary to auto-generate the filename, I think. 

[fdeng1, 2012/08/07 02:56:56]

As discussed offline, I override _GetArchiveName in each base test class which
returns the right archive name for that class.
On 2012/08/06 21:36:45, Nirnimesh wrote:

+    which means we only have 1 archive file per test class.
+    Override this function to allow other names.
+
+    Returns:
+      An archive name generated using test class name.
+    """
+    # Use class name as archive name.
+    return self.id().split('.')[1] + '.wpr'
 
   def _GetDeepMemoryProfileEnv(self, env_name, converter, default):
     """Returns a converted environment variable for the Deep Memory Profiler.
 
     Args:
       env_name: A string name of an environment variable.
       converter: A function taking a string to convert an environment variable.
       default: A value used if the environment variable is not specified.
 
     Returns:
@@ skipping 21 matching lines @@
     # The same with setUp, but need to fetch the environment variable since
     # ExtraChromeFlags is called before setUp.
     deep_memory_profile = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE', bool, self._DEEP_MEMORY_PROFILE)
 
     # Ensure Chrome enables remote debugging on port 9222.  This is required to
     # interact with Chrome's remote inspector.
     extra_flags = ['--remote-debugging-port=9222']
     if deep_memory_profile:
       extra_flags.append('--no-sandbox')
-    return (perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags)
+    if self._use_wpr:
+      extra_flags.extend(ChromeEndureReplay.CHROME_FLAGS)
+    return perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags
 
   def _OnTimelineEvent(self, event_info):
     """Invoked by the Remote Inspector Client when a timeline event occurs.
 
     Args:
       event_info: A dictionary containing raw information associated with a
          timeline event received from Chrome's remote inspector.  Refer to
          chrome/src/third_party/WebKit/Source/WebCore/inspector/Inspector.json
          for the format of this dictionary.
     """
@@ skipping 339 matching lines @@
     try:
       element = self._GetElement(driver.find_element_by_xpath, xpath)
       element.click()
     except (selenium.common.exceptions.StaleElementReferenceException,
             selenium.common.exceptions.TimeoutException) as e:
       logging.exception('WebDriver exception: %s' % e)
       return False
 
     return True
 
+  def _StartReplayServerIfNecessary(self):
+    """Start replay server if necessary.
+
+    This method needs to be called BEFORE any connection (which is supposed
+    to go through the Web Page Replay server) occurs.
+    """
+    if self._use_wpr and not self._is_record_mode:
+      archive_name = self._GenArchiveName()
+      self._wpr_server = ChromeEndureReplay.ReplayServer(archive_name)
+      self._wpr_server.StartServer()
+      logging.info('Web Page Replay server has started in Replay mode.')
+
+  def _StopReplayServerIfNecessary(self):
+    """Stop the Web Page Replay server if necessary.
+
+    This method has to be called AFTER all network connections which go
+    through Web Page Replay server have shut down. Otherwise the
+    Web Page Replay server will hang to wait for them. A good
+    place is to call it at the end of the teardown process.
+    """
+    if self._use_wpr and not self._is_record_mode:
+      self._wpr_server.StopServer()
+      logging.info('The Web Page Replay server stopped.')
+
 
 class ChromeEndureControlTest(ChromeEndureBaseTest):
   """Control tests for Chrome Endure."""
 
   _WEBAPP_NAME = 'Control'
   _TAB_TITLE_SUBSTRING = 'Chrome Endure Control Test'
 
   def testControlAttachDetachDOMTree(self):
     """Continually attach and detach a DOM tree from a basic document."""
     test_description = 'AttachDetachDOMTree'
@@ skipping 563 matching lines @@
       except (pyauto_errors.JSONInterfaceError,
               pyauto_errors.JavascriptRuntimeError):
         self._num_errors += 1
 
       time.sleep(1)
 
     self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING,
                         test_description, scenario)
 
 
+class ChromeEndureReplay(object):
+  """Run Chrome Endure tests with network simulation via Web Page Replay."""
+
+  _PATHS = {
+      'archive':
+      'src/chrome/test/data/pyauto_private/webpagereplay/{archive_name}',
+      'scripts':
+      'src/chrome/test/data/chrome_endure/webpagereplay/wpr_deterministic.js',
+      }
+  CHROME_FLAGS = webpagereplay.CHROME_FLAGS
+
+  @classmethod
+  def Path(cls, key, **kwargs):
+    return perf.FormatChromePath(cls._PATHS[key], **kwargs)
+
+  @classmethod
+  def ReplayServer(cls, archive_name):
+    """Creat a replay server."""
+    # Inject customized scripts for Google webapps.
+    # See the java script file for details.

[Nirnimesh, 2012/08/06 21:36:45]

java script -> javascript

[fdeng1, 2012/08/07 02:56:56]

Done.

+    scripts = cls.Path('scripts')
+    if not os.path.exists(scripts):
+      raise webpagereplay.ReplayNotFoundError('injected scripts', scripts)
+    replay_options = ['--inject_scripts', scripts]
+    archive_path = cls.Path('archive', archive_name=archive_name)
+    return webpagereplay.ReplayServer(archive_path, replay_options)
+
+
 if __name__ == '__main__':
   pyauto_functional.Main()