Run Chrome Endure tests with network simulation via Web Page Replay.

chrome/test/functional/perf_endure.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance tests for Chrome Endure (long-running perf tests on Chrome).
 
 This module accepts the following environment variable inputs:
   TEST_LENGTH: The number of seconds in which to run each test.
   PERF_STATS_INTERVAL: The number of seconds to wait in-between each sampling
@@ skipping 15 matching lines @@
 import time
 
 import perf
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 import pyauto_errors
 import pyauto_utils
 import remote_inspector_client
 import selenium.common.exceptions
 from selenium.webdriver.support.ui import WebDriverWait
+import webpagereplay
 
 
 class NotSupportedEnvironmentError(RuntimeError):
   """Represent an error raised since the environment (OS) is not supported."""
   pass
 
 class ChromeEndureBaseTest(perf.BasePerfTest):
   """Implements common functionality for all Chrome Endure tests.
 
   All Chrome Endure test classes should inherit from this class.
@@ skipping 526 matching lines @@
                         test_description, lambda: scenario(driver))
 
 
 class ChromeEndureGmailTest(ChromeEndureBaseTest):
   """Long-running performance tests for Chrome using Gmail."""
 
   _WEBAPP_NAME = 'Gmail'
   _TAB_TITLE_SUBSTRING = 'Gmail'
   _FRAME_XPATH = 'id("canvas_frame")'
 
-  def setUp(self):
-    ChromeEndureBaseTest.setUp(self)
+  def ExtraChromeFlags(self):

[Nirnimesh, 2012/07/18 19:41:18]

I wish you add this functionality to ChromeEndureBaseTest, rather than just in
the gmail test.

[fdeng1, 2012/07/24 00:05:50]

Done. 

I also moved the code that starts/shuts down WPR to
ChromeEndureBaseTest(_StartReplayServerIfNecessary and
_StopReplayServerIfNecessary).Would you please take a look?

+    flags = super(ChromeEndureGmailTest, self).ExtraChromeFlags()
+    # Add flags for the Web Page Replay server.
+    if 'USE_WPR_FOR_ENDURE' in os.environ:
+      return flags + ChromeEndureWebPageReplay.CHROME_FLAGS
+    return flags
 
+  def _GmailSetUp(self, test_name):
+    # Start the Web Page Replay server.
+    if 'USE_WPR_FOR_ENDURE' in os.environ:
+      mode = 'Record' if 'ENDURE_WPR_RECORD' in os.environ else 'Replay'
+      logging.info('Starting the Web Page Replay server: in %s mode', mode)
+
+      self.wpr_server = ChromeEndureWebPageReplay().GetReplayServer(test_name)
+      self.wpr_server.StartServer()
+      logging.info('The Web Page Replay server started.')
     # Log into a test Google account and open up Gmail.
     self._LoginToGoogleAccount(account_key='test_google_account_gmail')
     self.NavigateToURL('http://www.gmail.com')
     loaded_tab_title = self.GetActiveTabTitle()
     self.assertTrue(self._TAB_TITLE_SUBSTRING in loaded_tab_title,
                     msg='Loaded tab title does not contain "%s": "%s"' %
                         (self._TAB_TITLE_SUBSTRING, loaded_tab_title))
 
     self._driver = self.NewWebDriver()
     # Any call to wait.until() will raise an exception if the timeout is hit.
     # TODO(dennisjeffrey): Remove the need for webdriver's wait using the new
     # DOM mutation observer mechanism.
     self._wait = WebDriverWait(self._driver, timeout=60)
 
     # Wait until Gmail's 'canvas_frame' loads and the 'Inbox' link is present.
     # TODO(dennisjeffrey): Check with the Gmail team to see if there's a better
     # way to tell when the webpage is ready for user interaction.
     self._wait.until(
         self._SwitchToCanvasFrame)  # Raises exception if the timeout is hit.
     # Wait for the inbox to appear.
     self.WaitForDomNode('//a[starts-with(@title, "Inbox")]',
                         frame_xpath=self._FRAME_XPATH)
 
+  def tearDown(self):
+    super(ChromeEndureGmailTest, self).tearDown()
+    # Stop the Web Page Replay server. This has to be done AFTER
+    # the parent's tearDown has run. Otherwise the Web Page Replay server
+    # will hang to wait for all connection to close.
+    if 'USE_WPR_FOR_ENDURE' in os.environ:
+      logging.info('Stopping The Web Page Replay server.')
+      self.wpr_server.StopServer()
+      logging.info('The Web Page Replay server stopped.')
+
   def _SwitchToCanvasFrame(self, driver):
     """Switch the WebDriver to Gmail's 'canvas_frame', if it's available.
 
     Args:
       driver: A selenium.webdriver.remote.webdriver.WebDriver object.
 
     Returns:
       True, if the switch to Gmail's 'canvas_frame' is successful, or
       False if not.
     """
@@ skipping 59 matching lines @@
     else:
       logging.warning('Could not identify latency value.')
 
   def testGmailComposeDiscard(self):
     """Continuously composes/discards an e-mail before sending.
 
     This test continually composes/discards an e-mail using Gmail, and
     periodically gathers performance stats that may reveal memory bloat.
     """
     test_description = 'ComposeDiscard'
+    self._GmailSetUp(test_description)
 
     # TODO(dennisjeffrey): Remove following line once crosbug.com/32357 is
     # fixed.
     self._test_length_sec = 60 * 60 * 5  # Run test for 5 hours.
 
     def scenario():
       # Click the "Compose" button, enter some text into the "To" field, enter
       # some text into the "Subject" field, then click the "Discard" button to
       # discard the message.
       compose_xpath = '//div[text()="COMPOSE"]'
@@ skipping 30 matching lines @@
 
   # TODO(dennisjeffrey): Remove this test once the Gmail team is done analyzing
   # the results after the test runs for a period of time.
   def testGmailComposeDiscardSleep(self):
     """Like testGmailComposeDiscard, but sleeps for 30s between iterations.
 
     This is a temporary test requested by the Gmail team to compare against the
     results from testGmailComposeDiscard above.
     """
     test_description = 'ComposeDiscardSleep'
+    self._GmailSetUp(test_description)
 
     # TODO(dennisjeffrey): Remove following line once crosbug.com/32357 is
     # fixed.
     self._test_length_sec = 60 * 60 * 5  # Run test for 5 hours.
 
     def scenario():
       # Click the "Compose" button, enter some text into the "To" field, enter
       # some text into the "Subject" field, then click the "Discard" button to
       # discard the message.  Finally, sleep for 30 seconds.
       compose_xpath = '//div[text()="COMPOSE"]'
@@ skipping 34 matching lines @@
                         test_description, scenario)
 
   def testGmailAlternateThreadlistConversation(self):
     """Alternates between threadlist view and conversation view.
 
     This test continually clicks between the threadlist (Inbox) and the
     conversation view (e-mail message view), and periodically gathers
     performance stats that may reveal memory bloat.
     """
     test_description = 'ThreadConversation'
+    self._GmailSetUp(test_description)
 
     def scenario():
       # Click an e-mail to see the conversation view, wait 1 second, click the
       # "Inbox" link to see the threadlist, wait 1 second.
 
       # Find the first thread (e-mail) identified by a "span" tag that contains
       # an "email" attribute.  Then click it and wait for the conversation view
       # to appear (assumed to be visible when a particular div exists on the
       # page).
       thread_xpath = '//span[@email]'
@@ skipping 22 matching lines @@
     self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING,
                         test_description, scenario)
 
   def testGmailAlternateTwoLabels(self):
     """Continuously alternates between two labels.
 
     This test continually clicks between the "Inbox" and "Sent Mail" labels,
     and periodically gathers performance stats that may reveal memory bloat.
     """
     test_description = 'AlternateLabels'
+    self._GmailSetUp(test_description)
 
     def scenario():
       # Click the "Sent Mail" label, wait for 1 second, click the "Inbox" label,
       # wait for 1 second.
 
       # Click the "Sent Mail" label, then wait for the tab title to be updated
       # with the substring "sent".
       sent_xpath = '//a[starts-with(text(), "Sent Mail")]'
       self.WaitForDomNode(sent_xpath, frame_xpath=self._FRAME_XPATH)
       sent = self._GetElement(self._driver.find_element_by_xpath, sent_xpath)
@@ skipping 21 matching lines @@
 
   def testGmailExpandCollapseConversation(self):
     """Continuously expands/collapses all messages in a conversation.
 
     This test opens up a conversation (e-mail thread) with several messages,
     then continually alternates between the "Expand all" and "Collapse all"
     views, while periodically gathering performance stats that may reveal memory
     bloat.
     """
     test_description = 'ExpandCollapse'
+    self._GmailSetUp(test_description)
 
     # Enter conversation view for a particular thread.
     thread_xpath = '//span[@email]'
     self.WaitForDomNode(thread_xpath, frame_xpath=self._FRAME_XPATH)
     thread = self._GetElement(self._driver.find_element_by_xpath, thread_xpath)
     thread.click()
     self.WaitForDomNode('//div[text()="Click here to "]',
                         frame_xpath=self._FRAME_XPATH)
 
     def scenario():
@@ skipping 210 matching lines @@
       except (pyauto_errors.JSONInterfaceError,
               pyauto_errors.JavascriptRuntimeError):
         self._num_errors += 1
 
       time.sleep(1)
 
     self._RunEndureTest(self._WEBAPP_NAME, self._TAB_TITLE_SUBSTRING,
                         test_description, scenario)
 
 
+class ChromeEndureWebPageReplay(perf.WebPageReplay):
+  """Set up Web Page Replay server.
+
+  This class inherits from perf.WebPageReplay to allow different settings
+  for the Web Page Replay server.
+
+  Environment Variables:
+    ENDURE_WPR_RECORD:        If set, puts Web Page Replay in record mode
+                              instead of replay.
+    ENDURE_WPR_REPLAY_DIR:    Path to alternate Web Page Replay source
+                              (for development).
+    ENDUER_ WPR_ARCHIVE_PATH: Path to alternate archive file.
+                              E.g. '/tmp/foo.wpr'.
+  """
+  _PATHS = {
+      'archive': 'src/data/endure/webpagereplay/{test_name}.wpr',
+      'replay': 'src/third_party/webpagereplay',
+      'logs': 'src/chrome_endure_webpagereplay_logs',
+      'scripts': 'src/chrome/test/data/chrome_endure/deterministic.js'
+      }
+
+  CHROME_FLAGS = [
+      '--host-resolver-rules=MAP * %s' % webpagereplay.REPLAY_HOST,

[Nirnimesh, 2012/07/18 19:41:18]

If it works with it, I think you should retain the EXCLUDE localhost part, so
that it does not interfere when using the local testserver for serving local
data files.

[fdeng1, 2012/07/24 00:05:50]

I restructured perf.WebPageReplay. Now we have perf.BaseWebPageReplay and two
subclasses perf.PerWebPageReplay and perf_endure.ChromeEndureWebPageReplay. This
is to allow different settings for different tests.
 
"EXCLUDE localhost" is retained in perf.BaseWebPageReplay.

On 2012/07/18 19:41:18, Nirnimesh wrote:

+      '--testing-fixed-http-port=%s' % webpagereplay.HTTP_PORT,
+      '--testing-fixed-https-port=%s' % webpagereplay.HTTPS_PORT,
+      '--log-level=0',

[Nirnimesh, 2012/07/18 19:41:18]

remove this and other flags that are not necessary.

[fdeng1, 2012/07/24 00:05:50]

Done.
Please take a look at __init__ in perf.BaseWebPageReplay, perf.PerWebPageReplay
and perf_endure.ChromeEndureWebPageReplay.

+      '--disable-background-networking',
+      '--enable-logging',
+      '--ignore-certificate-errors',
+      '--no-first-run',
+      '--no-proxy-server',
+      ]
+
+  def __init__(self):
+    self.archive_path = os.environ.get('ENDURE_WPR_ARCHIVE_PATH')
+    self.replay_dir = os.environ.get('ENDURE_WPR_REPLAY_DIR',
+                                     self.Path('replay'))
+    self.is_record_mode = 'ENDURE_WPR_RECORD' in os.environ
+    if self.is_record_mode:
+      self._num_iterations = 1
+
+  def GetReplayServer(self, test_name):
+    """Create the Web Page Replay server.
+
+    This function overrides the one in perf.WebpageReplay to pass a customized
+    deterministic.js to Web Page Replay server. This one differs from the
+    default one as it uses a different value for time_seed. See the file
+    (src/chrome/test/data/chrome_endure/deterministic.js) for
+    how time_seed works.
+
+    Args:
+      test_name: the test name, should be identical for each test.
+    Returns:
+      An instance of webpagereplay.ReplayServer.
+    """
+
+    replay_options = []
+    replay_options.append('--no-dns_forwarding')
+    if self.is_record_mode:
+      replay_options.append('--record')
+
+    assert os.path.exists(self.Path('scripts')), \
+        'The scripts to be injected do not exist: %s' % self.Path('scripts')
+    replay_options.append('--inject_scripts=%s' % self.Path('scripts'))
+
+    return webpagereplay.ReplayServer(
+        self.replay_dir,
+        self._ArchivePath(test_name),
+        self.Path('logs'),
+        replay_options)
+
+
 if __name__ == '__main__':
   pyauto_functional.Main()