Run Chrome Endure tests with network simulation via Web Page Replay.

chrome/test/functional/perf_endure.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance tests for Chrome Endure (long-running perf tests on Chrome).
 
 This module accepts the following environment variable inputs:
   TEST_LENGTH: The number of seconds in which to run each test.
   PERF_STATS_INTERVAL: The number of seconds to wait in-between each sampling
@@ skipping 15 matching lines @@
 import time
 
 import perf
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 import pyauto_errors
 import pyauto_utils
 import remote_inspector_client
 import selenium.common.exceptions
 from selenium.webdriver.support.ui import WebDriverWait
+import webpagereplay
 
 
 class NotSupportedEnvironmentError(RuntimeError):
   """Represent an error raised since the environment (OS) is not supported."""
   pass
 
+
 class ChromeEndureBaseTest(perf.BasePerfTest):
   """Implements common functionality for all Chrome Endure tests.
 
   All Chrome Endure test classes should inherit from this class.
   """
 
   _DEFAULT_TEST_LENGTH_SEC = 60 * 60 * 6  # Tests run for 6 hours.
   _GET_PERF_STATS_INTERVAL = 60 * 5  # Measure perf stats every 5 minutes.
   # TODO(dennisjeffrey): Do we still need to tolerate errors?
   _ERROR_COUNT_THRESHOLD = 50  # Number of ChromeDriver errors to tolerate.
   _DEEP_MEMORY_PROFILE = False
   _DEEP_MEMORY_PROFILE_INTERVAL = _GET_PERF_STATS_INTERVAL
   _DEEP_MEMORY_PROFILE_SAVE = False
 
   _DMPROF_DIR_PATH = os.path.join(
       os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
       'tools', 'deep_memory_profiler')
 
   _DMPROF_SCRIPT_PATH = os.path.join(_DMPROF_DIR_PATH, 'dmprof')
 
   _CHROME_BIN_PATH = os.path.join(perf.BasePerfTest.BrowserPath(), 'chrome')
 
   def setUp(self):
+    # The environment variable for the usage of Web Page Replay.
+    # It must be parsed before perf.BasePerfTest.setUp()
+    self._use_wpr = 'ENDURE_NO_WPR' not in os.environ
+
     # The environment variables for the Deep Memory Profiler must be set
     # before perf.BasePerfTest.setUp() to inherit them to Chrome.
     self._deep_memory_profile = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE', bool, self._DEEP_MEMORY_PROFILE)
 
     self._deep_memory_profile_interval = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE_INTERVAL', int, self._DEEP_MEMORY_PROFILE_INTERVAL)
 
     if self._deep_memory_profile:
       if not self.IsLinux():
@@ skipping 98 matching lines @@
     # The same with setUp, but need to fetch the environment variable since
     # ExtraChromeFlags is called before setUp.
     deep_memory_profile = self._GetDeepMemoryProfileEnv(
         'DEEP_MEMORY_PROFILE', bool, self._DEEP_MEMORY_PROFILE)
 
     # Ensure Chrome enables remote debugging on port 9222.  This is required to
     # interact with Chrome's remote inspector.
     extra_flags = ['--remote-debugging-port=9222']
     if deep_memory_profile:
       extra_flags.append('--no-sandbox')
-    return (perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags)
+    if self._use_wpr:
+      extra_flags.extend(ChromeEndureReplay.CHROME_FLAGS)
+    return perf.BasePerfTest.ExtraChromeFlags(self) + extra_flags
 
   def _OnTimelineEvent(self, event_info):
     """Invoked by the Remote Inspector Client when a timeline event occurs.
 
     Args:
       event_info: A dictionary containing raw information associated with a
          timeline event received from Chrome's remote inspector.  Refer to
          chrome/src/third_party/WebKit/Source/WebCore/inspector/Inspector.json
          for the format of this dictionary.
     """
@@ skipping 339 matching lines @@
     try:
       element = self._GetElement(driver.find_element_by_xpath, xpath)
       element.click()
     except (selenium.common.exceptions.StaleElementReferenceException,
             selenium.common.exceptions.TimeoutException) as e:
       logging.exception('WebDriver exception: %s' % e)
       return False
 
     return True
 
+  def _StartReplayServerIfNecessary(self, archive_name):
+    """Start the Web Page Replay server if ENDURE_NO_WPR is not set.
+
+    This method needs to be called BEFORE any connection (which are supposed
+    to go through the Web Page Replay server) occurs.
+
+    Args:
+      archive_name: a string representing the name of the web page replay
+                    archive.
+    """
+    if self._use_wpr:
+      self._wpr_server = ChromeEndureReplay.ReplayServer(archive_name)
+      mode = 'Record' if self._wpr_server.is_record_mode else 'Replay'
+      logging.info('Starting the Web Page Replay server: in %s mode', mode)
+      self._wpr_server.StartServer()
+      logging.info('The Web Page Replay server started.')

[Nirnimesh, 2012/08/01 01:03:00]

Too much logging. Please pick between this line and 564

[fdeng1, 2012/08/01 17:14:57]

Done.

+
+  def _StopReplayServerIfNecessary(self):
+    """Stop the Web Page Replay server if ENDURE_NO_WPR is not set.
+
+    This method has to be called AFTER all network connections which go
+    through Web Page Replay server have shut down. Otherwise the
+    Web Page Replay server will hang to wait for them. A good
+    place is to call it at the end of the teardown process.
+    """
+    if self._use_wpr:
+      logging.info('Stopping The Web Page Replay server.')
+      self._wpr_server.StopServer()
+      logging.info('The Web Page Replay server stopped.')
+
 
 class ChromeEndureControlTest(ChromeEndureBaseTest):
   """Control tests for Chrome Endure."""
 
   _WEBAPP_NAME = 'Control'
   _TAB_TITLE_SUBSTRING = 'Chrome Endure Control Test'
 
   def testControlAttachDetachDOMTree(self):
     """Continually attach and detach a DOM tree from a basic document."""
     test_description = 'AttachDetachDOMTree'
@@ skipping 37 matching lines @@
                         test_description, lambda: scenario(driver))
 
 
 class ChromeEndureGmailTest(ChromeEndureBaseTest):
   """Long-running performance tests for Chrome using Gmail."""
 
   _WEBAPP_NAME = 'Gmail'
   _TAB_TITLE_SUBSTRING = 'Gmail'
   _FRAME_XPATH = 'id("canvas_frame")'
 
-  def setUp(self):
-    ChromeEndureBaseTest.setUp(self)
+  def _GmailSetUp(self, archive_name):
+    """Set up before each test runs."""
+    self._StartReplayServerIfNecessary(archive_name)

[Nirnimesh, 2012/08/01 01:03:00]

I thought this was going to be done in ChromeEndureBaseTest so that it can be
used in _any_ endure test.

[fdeng1, 2012/08/01 17:14:57]

When starting a replay server, an "archive_name" need to be passed to
ChromeEndureReplay.ReplayServer(cls, archive_name). For now, archive_name is
unique for each test because each test is using its own archive. I was trying to
put self._StartReplayServerIfNecessary(archive_name) is in
ChromeEndureBaseTest.setUp() or ChromeEndureGmailTest.setUp(), but I realized I
was unable to know which test is about run in the setUp phase and as a result I
was unable to pick a unique archive_name for that test.  Is there a way to know
which test is about to run during the setUp phase? 

(I was also thinking put all archives into a big one which will be shared by all
tests. But the performance of WPR degrades a lot when the archive is big.)

On 2012/08/01 01:03:00, Nirnimesh wrote:

[dennis_jeffrey, 2012/08/01 18:35:54]

self.id(), as discussed offline
We'll likely be able to remove all of these _XSetUp() functions.

[fdeng1, 2012/08/01 18:37:31]

I just talked to Dennis and we came up a plan to combine archives for the same
webapp into one single archive, i.e. we will have Gmail.wpr, Plus.wpr, Docs.wpr.

I will put _GmailSetUp in setUp() function in a coming patch.

On 2012/08/01 17:14:57, fdeng1 wrote:

 
     # Log into a test Google account and open up Gmail.
     self._LoginToGoogleAccount(account_key='test_google_account_gmail')
     self.NavigateToURL('http://www.gmail.com')
     loaded_tab_title = self.GetActiveTabTitle()
     self.assertTrue(self._TAB_TITLE_SUBSTRING in loaded_tab_title,
                     msg='Loaded tab title does not contain "%s": "%s"' %
                         (self._TAB_TITLE_SUBSTRING, loaded_tab_title))
 
     self._driver = self.NewWebDriver()
     # Any call to wait.until() will raise an exception if the timeout is hit.
     # TODO(dennisjeffrey): Remove the need for webdriver's wait using the new
     # DOM mutation observer mechanism.
     self._wait = WebDriverWait(self._driver, timeout=60)
 
     # Wait until Gmail's 'canvas_frame' loads and the 'Inbox' link is present.
     # TODO(dennisjeffrey): Check with the Gmail team to see if there's a better
     # way to tell when the webpage is ready for user interaction.
     self._wait.until(
         self._SwitchToCanvasFrame)  # Raises exception if the timeout is hit.
     # Wait for the inbox to appear.
     self.WaitForDomNode('//a[starts-with(@title, "Inbox")]',
                         frame_xpath=self._FRAME_XPATH)
 
+  def tearDown(self):
+    super(ChromeEndureGmailTest, self).tearDown()
+    # Stop the Web Page Replay server after all connections to WPR closed.
+    self._StopReplayServerIfNecessary()
+
   def _SwitchToCanvasFrame(self, driver):
     """Switch the WebDriver to Gmail's 'canvas_frame', if it's available.
 
     Args:
       driver: A selenium.webdriver.remote.webdriver.WebDriver object.
 
     Returns:
       True, if the switch to Gmail's 'canvas_frame' is successful, or
       False if not.
     """
@@ skipping 59 matching lines @@
     else:
       logging.warning('Could not identify latency value.')
 
   def testGmailComposeDiscard(self):
     """Continuously composes/discards an e-mail before sending.
 
     This test continually composes/discards an e-mail using Gmail, and
     periodically gathers performance stats that may reveal memory bloat.
     """
     test_description = 'ComposeDiscard'
+    self._GmailSetUp(test_description)

[Nirnimesh, 2012/08/01 01:03:00]

Why not do this it setUp instead of calling before each test?

[fdeng1, 2012/08/01 17:14:57]

Please see the above comment.

On 2012/08/01 01:03:00, Nirnimesh wrote:

 
     # TODO(dennisjeffrey): Remove following line once crosbug.com/32357 is
     # fixed.
     self._test_length_sec = 60 * 60 * 5  # Run test for 5 hours.
 
     def scenario():
       # Click the "Compose" button, enter some text into the "To" field, enter
       # some text into the "Subject" field, then click the "Discard" button to
       # discard the message.
       compose_xpath = '//div[text()="COMPOSE"]'
@@ skipping 31 matching lines @@
 
   # TODO(dennisjeffrey): Remove this test once the Gmail team is done analyzing
   # the results after the test runs for a period of time.
   def testGmailComposeDiscardSleep(self):
     """Like testGmailComposeDiscard, but sleeps for 30s between iterations.
 
     This is a temporary test requested by the Gmail team to compare against the
     results from testGmailComposeDiscard above.
     """
     test_description = 'ComposeDiscardSleep'
+    # Use the same archive_name as testGmailComposeDiscard uses.
+    self._GmailSetUp('ComposeDiscard')
 
     # TODO(dennisjeffrey): Remove following line once crosbug.com/32357 is
     # fixed.
     self._test_length_sec = 60 * 60 * 5  # Run test for 5 hours.
 
     def scenario():
       # Click the "Compose" button, enter some text into the "To" field, enter
       # some text into the "Subject" field, then click the "Discard" button to
       # discard the message.  Finally, sleep for 30 seconds.
       compose_xpath = '//div[text()="COMPOSE"]'
@@ skipping 35 matching lines @@
                         frame_xpath=self._FRAME_XPATH)
 
   def testGmailAlternateThreadlistConversation(self):
     """Alternates between threadlist view and conversation view.
 
     This test continually clicks between the threadlist (Inbox) and the
     conversation view (e-mail message view), and periodically gathers
     performance stats that may reveal memory bloat.
     """
     test_description = 'ThreadConversation'
+    self._GmailSetUp(test_description)
 
     def scenario():
       # Click an e-mail to see the conversation view, wait 1 second, click the
       # "Inbox" link to see the threadlist, wait 1 second.
 
       # Find the first thread (e-mail) identified by a "span" tag that contains
       # an "email" attribute.  Then click it and wait for the conversation view
       # to appear (assumed to be visible when a particular div exists on the
       # page).
       thread_xpath = '//span[@email]'
@@ skipping 23 matching lines @@
                         test_description, scenario,
                         frame_xpath=self._FRAME_XPATH)
 
   def testGmailAlternateTwoLabels(self):
     """Continuously alternates between two labels.
 
     This test continually clicks between the "Inbox" and "Sent Mail" labels,
     and periodically gathers performance stats that may reveal memory bloat.
     """
     test_description = 'AlternateLabels'
+    self._GmailSetUp(test_description)
 
     def scenario():
       # Click the "Sent Mail" label, wait for 1 second, click the "Inbox" label,
       # wait for 1 second.
 
       # Click the "Sent Mail" label, then wait for the tab title to be updated
       # with the substring "sent".
       sent_xpath = '//a[starts-with(text(), "Sent Mail")]'
       self.WaitForDomNode(sent_xpath, frame_xpath=self._FRAME_XPATH)
       sent = self._GetElement(self._driver.find_element_by_xpath, sent_xpath)
@@ skipping 22 matching lines @@
 
   def testGmailExpandCollapseConversation(self):
     """Continuously expands/collapses all messages in a conversation.
 
     This test opens up a conversation (e-mail thread) with several messages,
     then continually alternates between the "Expand all" and "Collapse all"
     views, while periodically gathering performance stats that may reveal memory
     bloat.
     """
     test_description = 'ExpandCollapse'
+    self._GmailSetUp(test_description)
 
     # Enter conversation view for a particular thread.
     thread_xpath = '//span[@email]'
     self.WaitForDomNode(thread_xpath, frame_xpath=self._FRAME_XPATH)
     thread = self._GetElement(self._driver.find_element_by_xpath, thread_xpath)
     thread.click()
     self.WaitForDomNode('//div[text()="Click here to "]',
                         frame_xpath=self._FRAME_XPATH)
 
     def scenario():
@@ skipping 30 matching lines @@
                         test_description, scenario,
                         frame_xpath=self._FRAME_XPATH)
 
 
 class ChromeEndureDocsTest(ChromeEndureBaseTest):
   """Long-running performance tests for Chrome using Google Docs."""
 
   _WEBAPP_NAME = 'Docs'
   _TAB_TITLE_SUBSTRING = 'Google Drive'
 
-  def setUp(self):
-    ChromeEndureBaseTest.setUp(self)
+  def _DocsSetUp(self, archive_name):
+    """Set up before each test runs."""
+    self._StartReplayServerIfNecessary(archive_name)
 
     # Log into a test Google account and open up Google Docs.
     self._LoginToGoogleAccount()
     self.NavigateToURL('http://docs.google.com')
     self.assertTrue(
         self.WaitUntil(lambda: self._TAB_TITLE_SUBSTRING in
                                self.GetActiveTabTitle(),
                        timeout=60, expect_retval=True, retry_sleep=1),
         msg='Timed out waiting for Docs to load. Tab title is: %s' %
             self.GetActiveTabTitle())
 
     self._driver = self.NewWebDriver()
 
+  def tearDown(self):
+    super(ChromeEndureDocsTest, self).tearDown()
+    # Stop the Web Page Replay server after all connections to it closed.
+    self._StopReplayServerIfNecessary()
+
   def testDocsAlternatelyClickLists(self):
     """Alternates between two different document lists.
 
     This test alternately clicks the "Shared with me" and "My Drive" buttons in
     Google Docs, and periodically gathers performance stats that may reveal
     memory bloat.
     """
     test_description = 'AlternateLists'
+    self._DocsSetUp(test_description)
 
     def scenario():
       # Click the "Shared with me" button, wait for 1 second, click the
       # "My Drive" button, wait for 1 second.
 
       # Click the "Shared with me" button and wait for a div to appear.
       if not self._ClickElementByXpath(
           self._driver, '//span[starts-with(text(), "Shared with me")]'):
         self._num_errors += 1
       self.WaitForDomNode('//div[text()="Share date"]')
@@ skipping 18 matching lines @@
     self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING,
                         test_description, scenario)
 
 
 class ChromeEndurePlusTest(ChromeEndureBaseTest):
   """Long-running performance tests for Chrome using Google Plus."""
 
   _WEBAPP_NAME = 'Plus'
   _TAB_TITLE_SUBSTRING = 'Google+'
 
-  def setUp(self):
-    ChromeEndureBaseTest.setUp(self)
+  def _PlusSetUp(self, archive_name):
+    """Set up before each test runs."""
+    self._StartReplayServerIfNecessary(archive_name)
 
     # Log into a test Google account and open up Google Plus.
     self._LoginToGoogleAccount()
     self.NavigateToURL('http://plus.google.com')
     loaded_tab_title = self.GetActiveTabTitle()
     self.assertTrue(self._TAB_TITLE_SUBSTRING in loaded_tab_title,
                     msg='Loaded tab title does not contain "%s": "%s"' %
                         (self._TAB_TITLE_SUBSTRING, loaded_tab_title))
 
     self._driver = self.NewWebDriver()
 
+  def tearDown(self):
+    super(ChromeEndurePlusTest, self).tearDown()
+    # Stop the Web Page Replay server after all connections to it closed.
+    self._StopReplayServerIfNecessary()
+
   def testPlusAlternatelyClickStreams(self):
     """Alternates between two different streams.
 
     This test alternately clicks the "Friends" and "Family" buttons using
     Google Plus, and periodically gathers performance stats that may reveal
     memory bloat.
     """
     test_description = 'AlternateStreams'
+    self._PlusSetUp(test_description)
 
     # TODO(dennisjeffrey): Remove following line once crosbug.com/32357 is
     # fixed.
     self._test_length_sec = 60 * 60 * 3  # Run test for 3 hours.
 
     def scenario():
       # Click the "Friends" button, wait for 1 second, click the "Family"
       # button, wait for 1 second.
 
       # Click the "Friends" button and wait for a resulting div to appear.
@@ skipping 79 matching lines @@
       except (pyauto_errors.JSONInterfaceError,
               pyauto_errors.JavascriptRuntimeError):
         self._num_errors += 1
 
       time.sleep(1)
 
     self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING,
                         test_description, scenario)
 
 
+class ChromeEndureReplay(object):
+  """Run Chrome Endure tests with network simulation via Web Page Replay."""
+
+  _PATHS = {
+      'archive':
+      'src/chrome/test/data/pyauto_private/webpagereplay/{archive_name}.wpr',
+      'scripts':
+      'src/chrome/test/data/chrome_endure/webpagereplay/wpr_deterministic.js',
+      }
+  CHROME_FLAGS = webpagereplay.CHROME_FLAGS
+
+  @classmethod
+  def Path(cls, key, **kwargs):
+    return perf.FormatChromePath(cls._PATHS[key], **kwargs)
+
+  @classmethod
+  def ReplayServer(cls, archive_name):
+    """Creat a replay server."""
+    # Inject customized scripts for Google webapps.
+    # See the java script file for details.
+    scripts = cls.Path('scripts')
+    if not os.path.exists(scripts):
+      raise webpagereplay.ReplayNotFoundError('injected scripts', scripts)
+    replay_options = ['--inject_scripts', scripts]
+    archive_path = cls.Path('archive', archive_name=archive_name)
+    return webpagereplay.ReplayServer(archive_path, replay_options)
+
+
 if __name__ == '__main__':
   pyauto_functional.Main()