Make it easy to use Web Page Replay with non-page_cycler tests.

chrome/test/functional/webpagereplay.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Start and stop Web Page Replay."""
 
 import logging
-import optparse
 import os
-import shutil
 import signal
 import subprocess
-import sys
-import tempfile
 import time
 import urllib
 
 
-USAGE = '%s [options] CHROME_EXE TEST_NAME' % os.path.basename(sys.argv[0])
-USER_DATA_DIR = '{TEMP}/webpagereplay_utils-chrome'
-
-# The port numbers must match those in chrome/test/perf/page_cycler_test.cc.
 HTTP_PORT = 8080
 HTTPS_PORT = 8413
 REPLAY_HOST='127.0.0.1'
+CHROME_FLAGS = [
+    '--host-resolver-rules=MAP * %s,EXCLUDE localhost' % REPLAY_HOST,
+    '--testing-fixed-http-port=%s' % HTTP_PORT,
+    '--testing-fixed-https-port=%s' % HTTPS_PORT,
+    '--ignore-certificate-errors',
+    ]
+
+_BASE_DIR = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir))
+REPLAY_DIR = os.path.join(_BASE_DIR, 'src', 'third_party', 'webpagereplay')
+LOG_PATH = os.path.join(_BASE_DIR, 'src', 'webpagereplay_logs', 'logs.txt')
 
 
 class ReplayError(Exception):
   """Catch-all exception for the module."""
   pass
 
 class ReplayNotFoundError(ReplayError):
   pass
 
 class ReplayNotStartedError(ReplayError):
   pass
 
 
+def FormatChromiumPath(posix_path, **kwargs):

[slamm_google, 2012/07/26 23:46:59]

How about I move this to perf.py:PageCyclerReplay since it is not used here?

[fdeng1, 2012/07/27 00:45:10]

Yes, I agree it makes more sense to be that way. Maybe merge it to
"PageCyclerReplay.Path(cls, key, **kwargs)"?

The only downside I see is that perf_endure.py:ChromeEndureReplay will be using
it too so that we will have some duplicated code. I think it is fine.

+  """Convert a path relative to the Chromium root into an OS-specific path.
+
+  Args:
+    posix_path: a path string that may be a format().
+    kwargs: args for the format replacement.
+  Examples:
+    FormatChromiumPath('src/third_party/pylib')
+    FormatChromiumPath('src/third_party/{module_name}/__init__.py',
+                       module_name='pylib')
+  """
+  formated_path = posix_path.format(**kwargs)
+  path_parts = formated_path.split('/')
+  return os.path.join(_BASE_DIR, *path_parts)
+
+
 class ReplayServer(object):
   """Start and Stop Web Page Replay.
 
+  Web Page Replay is a proxy that can record and "replay" web pages with
+  simulated network characteristics -- without having to edit the pages
+  by hand. With WPR, tests can use "real" web content, and catch
+  performance issues that may result from introducing network delays and
+  bandwidth throttling.
+
   Example:
-     with ReplayServer(replay_dir, archive_path, log_dir, replay_options):
+     with ReplayServer(archive_path):
        self.NavigateToURL(start_url)
        self.WaitUntil(...)
+
+  Environment Variables (for development):
+    WPR_ARCHIVE_PATH: path to alternate archive file (e.g. '/tmp/foo.wpr').
+    WPR_RECORD: if set, puts Web Page Replay in record mode instead of replay.
+    WPR_REPLAY_DIR: path to alternate Web Page Replay source.
   """
-  LOG_FILE = 'log.txt'
-
-  def __init__(self, replay_dir, archive_path, log_dir, replay_options=None):
+  def __init__(self, archive_path, replay_options=None, replay_dir=None,
+               log_path=None):
     """Initialize ReplayServer.
 
     Args:
+      archive_path: a path to a specific WPR archive (required).
+      replay_options: a list of options strings to forward to replay.py.
       replay_dir: directory that has replay.py and related modules.
-      archive_path: either a directory that contains WPR archives or,
-          a path to a specific WPR archive.
-      log_dir: where to write log.txt.
-      replay_options: a list of options strings to forward to replay.py.
-    """
-    self.replay_dir = replay_dir
-    self.archive_path = archive_path
-    self.log_dir = log_dir
-    self.replay_options = replay_options if replay_options else []
+      log_path: a path to a log file.
+   """
+    self.archive_path = os.environ.get('WPR_ARCHIVE_PATH', archive_path)
+    self.replay_options = replay_options or []
+    self.replay_dir = os.environ.get('WPR_REPLAY_DIR', replay_dir or REPLAY_DIR)
+    self.log_path = log_path or LOG_PATH
 
-    self.log_name = os.path.join(self.log_dir, self.LOG_FILE)
+    if 'WPR_RECORD' in os.environ and '--record' not in self.replay_options:
+      replay_options.append('--record')

[fdeng1, 2012/07/26 20:51:19]

-> self.replay.options.append('--record')

[fdeng1, 2012/07/26 20:52:51]

sorry, should be self.replay_options.append(...)
On 2012/07/26 20:51:19, fdeng1 wrote:

[slamm_google, 2012/07/26 23:46:59]

Done.

+    self.is_record_mode = '--record' in self.replay_options
+    self._AddDefaultReplayOptions(self)

[fdeng1, 2012/07/26 20:51:19]

remove the argument "self"?

[slamm_google, 2012/07/26 23:46:59]

Done.

+
+    self.replay_py = os.path.join(self.replay_dir, 'replay.py')
+
+    self._CheckArgs()
+
     self.log_fh = None
     self.replay_process = None
 
-    self.wpr_py = os.path.join(self.replay_dir, 'replay.py')
-    if not os.path.exists(self.wpr_py):
-      raise ReplayNotFoundError('Path does not exist: %s' % self.wpr_py)
-    self.wpr_options = [
+  def _AddDefaultReplayOptions(self):
+    self.replay_options += [
         '--port', str(HTTP_PORT),
         '--ssl_port', str(HTTPS_PORT),
         '--use_closest_match',
-        # TODO(slamm): Add traffic shaping (requires root):
-        # '--net', 'fios',
+        '--no-dns_forwarding',
+        # '--net', 'fios',  # TODO(slamm): Add traffic shaping (requires root).
         ]
-    self.wpr_options.extend(self.replay_options)
+
+  def _CheckArgs(self):
+    assert self.archive_path, 'Archive path is not set.'
+    if self.is_record_mode:
+      archive_dir = os.path.dirname(self.archive_path)
+      assert os.path.exists(archive_dir), \
+          'Archive directory does not exist: %s' % self.archive_dir

[fdeng1, 2012/07/26 20:51:19]

self.archive_dir -> archive_dir

[slamm_google, 2012/07/26 23:46:59]

I dropped the asserts in favor of raising exceptions.

+    else:
+      assert os.path.exists(archive_path), \
+          'Archive file path does not exist: %s' % archive_path

[fdeng1, 2012/07/26 20:51:19]

In above two lines: archive_path -> self.archive_path

+    if not os.path.exists(self.replay_py):
+      raise ReplayNotFoundError('Path does not exist: %s' % self.replay_py)
 
   def _OpenLogFile(self):
-    if not os.path.exists(self.log_dir):
-      os.makedirs(self.log_dir)
-    return open(self.log_name, 'w')
+    log_dir = os.path.dirname(self.log_path)
+    if not os.path.exists(log_dir):
+      os.makedirs(log_dir)
+    return open(self.log_path, 'w')
 
   def IsStarted(self):
     """Checks to see if the server is up and running."""
     for _ in range(5):
       if self.replay_process.poll() is not None:
         # The process has exited.
         break
       try:
         up_url = '%s://localhost:%s/web-page-replay-generate-200'
         http_up_url = up_url % ('http', HTTP_PORT)
         https_up_url = up_url % ('https', HTTPS_PORT)
         if (200 == urllib.urlopen(http_up_url, None, {}).getcode() and
             200 == urllib.urlopen(https_up_url, None, {}).getcode()):
           return True
       except IOError:
         time.sleep(1)
     return False
 
   def StartServer(self):
     """Start Web Page Replay and verify that it started.
 
     Raises:
       ReplayNotStartedError if Replay start-up fails.
     """
-    cmd_line = [self.wpr_py]
-    cmd_line.extend(self.wpr_options)
+    cmd_line = [self.replay_py]
+    cmd_line.extend(self.replay_options)
     cmd_line.append(self.archive_path)
     self.log_fh = self._OpenLogFile()
     logging.debug('Starting Web-Page-Replay: %s', cmd_line)
     self.replay_process = subprocess.Popen(
       cmd_line, stdout=self.log_fh, stderr=subprocess.STDOUT)
     if not self.IsStarted():
       raise ReplayNotStartedError(
           'Web Page Replay failed to start. See the log file: ' + self.log_name)
 
   def StopServer(self):
     """Stop Web Page Replay."""
     if self.replay_process:
       logging.debug('Stopping Web-Page-Replay')
       # Use a SIGINT here so that it can do graceful cleanup.
       # Otherwise, we will leave subprocesses hanging.
       self.replay_process.send_signal(signal.SIGINT)
       self.replay_process.wait()
     if self.log_fh:
       self.log_fh.close()
 
   def __enter__(self):
     """Add support for with-statement."""
     self.StartServer()
 
   def __exit__(self, unused_exc_type, unused_exc_val, unused_exc_tb):
     """Add support for with-statement."""
     self.StopServer()