Add Web Page Replay test to page cycler.

tools/python/google/webpagereplay_utils.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import optparse
+import os
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import urllib
+
+

[tonyg, 2012/04/13 14:32:21]

A file level docstring would be helpful.

[slamm_google, 2012/04/13 23:43:36]

Done.

+USAGE = '%s [options] TEST_NAME' % os.path.basename(sys.argv[0])
+
+USER_DATA_DIR = '{TEMP}/webpagereplay_utils-chrome'
+
+class ReplayError(Exception):
+  """Catch-all exception for the module."""
+  pass
+
+class ReplayNotFoundError(Exception):
+  pass
+
+class ReplayNotStartedError(Exception):
+  pass
+
+
+class ReplayLauncher(object):
+  HTTP_PORT = 8080
+  HTTPS_PORT = 8413

[tonyg, 2012/04/13 14:32:21]

Looks like these have to match the ones hardcoded in the .cc file. Is there any
way to avoid the duplicate constants?

[slamm_google, 2012/04/13 23:43:36]

I moved these constants to the top of the file and added a comment about the
need to keep them in sync with chrome/test/perf/page_cycler_test.cc

+  UP_URL = 'http://localhost:%s/web-page-replay-generate-200' % HTTP_PORT
+  LOG_FILE = 'log.txt'
+
+  def __init__(self, replay_dir, archive_path, log_dir, replay_options=None):
+    """Initialize ReplayLauncher.
+
+    Args:
+      replay_dir: directory that has replay.py and related modules.
+      archive_path: either a directory that contains WPR archives or,
+          a path to a specific WPR archive.
+      log_dir: where to write log.txt.
+      replay_options: a list of options strings to forward to replay.py.
+    """
+    self.replay_dir = replay_dir
+    self.archive_path = archive_path
+    self.log_dir = log_dir
+    self.replay_options = replay_options if replay_options else []
+
+    self.log_name = os.path.join(self.log_dir, self.LOG_FILE)
+    self.log_fh = None
+    self.proxy_process = None
+
+    self.wpr_py = os.path.join(self.replay_dir, 'replay.py')
+    if not os.path.exists(self.wpr_py):
+      raise ReplayNotFoundError('Path does not exist: %s' % self.wpr_py)
+    self.wpr_options = [
+        '--port', str(self.HTTP_PORT),
+        '--ssl_port', str(self.HTTPS_PORT),
+        '--use_closest_match',
+        # TODO(slamm): Add traffic shaping (requires root):
+        # '--net', 'fios',
+        ]
+    self.wpr_options.extend(self.replay_options)
+
+  def OpenLogFile_(self):

[James Simonsen, 2012/04/12 23:14:19]

I think the _ goes on the other side in Python.

[slamm_google, 2012/04/13 23:43:36]

Done.

+    if not os.path.exists(self.log_dir):
+      os.makedirs(self.log_dir)
+    return open(self.log_name, 'w')
+
+  def StartServer(self):
+    cmd_line = [self.wpr_py]
+    cmd_line.extend(self.wpr_options)
+    # TODO(slamm): Support choosing archive on-the-fly.
+    cmd_line.append(self.archive_path)
+    self.log_fh = self.OpenLogFile_()
+    logging.debug('Starting Web-Page-Replay: %s', cmd_line)
+    self.proxy_process = subprocess.Popen(
+      cmd_line, stdout=self.log_fh, stderr=subprocess.STDOUT)
+    if not self.IsStarted():
+      raise ReplayNotStartedError(
+          'Web Page Replay failed to start. See the log file: ' + self.log_name)
+
+  def IsStarted(self):
+    """Checks to see if the server is up and running."""
+    for _ in range(5):
+      if self.proxy_process.poll() is not None:
+        # The process has exited.
+        break
+      try:
+        if 200 == urllib.urlopen(self.UP_URL, None, {}).getcode():
+          return True
+      except IOError:
+        time.sleep(1)
+    return False
+
+  def StopServer(self):
+    if self.proxy_process:
+      logging.debug('Stopping Web-Page-Replay')
+      # Use a SIGINT here so that it can do graceful cleanup.
+      # Otherwise, we will leave subprocesses hanging.
+      self.proxy_process.send_signal(signal.SIGINT)
+      self.proxy_process.wait()
+    if self.log_fh:
+      self.log_fh.close()
+
+
+class ChromiumPaths(object):
+  """Collect all the path handling together."""
+  PATHS = {
+      'archives':   'src/data/page_cycler/webpagereplay',
+      '.wpr':       'src/data/page_cycler/webpagereplay/{TEST_NAME}.wpr',
+      '.wpr_alt':   'src/tools/page_cycler/webpagereplay/tests/{TEST_NAME}.wpr',
+      'start.html': 'src/tools/page_cycler/webpagereplay/start.html',
+      'extension':  'src/tools/page_cycler/webpagereplay/extension',
+      'chrome':     'src/out/{TARGET}/chrome',

[James Simonsen, 2012/04/12 23:14:19]

This is Linux only. You might want to just make it an arg to pass in the chrome
binary.

[slamm_google, 2012/04/13 23:43:36]

Thanks for catching that. I see that runtest.py does do a fair amount of path
wrangling. A nice refactor would be to move that code into platform_utils.py.
However, for now I will pass the value from runtest.py and the
webpagetest_utils.py command-line.

+      'logs':       'src/webpagereplay_logs/{TEST_EXE_NAME}',
+      'replay':     'tools/build/third_party/webpagereplay',
+      }
+
+  def __init__(self, **replacements):
+    """Initialize ChromiumPaths.
+
+    Args:
+      replacements: a dict of format replacements for PATHS
+          (e.g. {'TARGET': 'Debug', 'TEST_NAME': '2012Q2'}).
+    """
+    module_dir = os.path.dirname(__file__)
+    self.base_dir = os.path.abspath(os.path.join(
+        module_dir, '..', '..', '..', '..'))
+    self.replacements = replacements
+
+  def __getitem__(self, key):
+    path_parts = [x.format(**self.replacements)
+                  for x in self.PATHS[key].split('/')]
+    return os.path.join(self.base_dir, *path_parts)
+
+
+def LaunchChromium(chromium_paths, test_name, is_dns_forwarded, use_auto):
+  REPLAY_HOST='127.0.0.1'
+  user_data_dir = USER_DATA_DIR.format(**{'TEMP': tempfile.gettempdir()})
+  chromium_args = [
+      chromium_paths['chrome'],
+      '--load-extension=%s' % chromium_paths['extension'],
+      '--testing-fixed-http-port=%s' % ReplayLauncher.HTTP_PORT,
+      '--testing-fixed-https-port=%s' % ReplayLauncher.HTTPS_PORT,

[tonyg, 2012/04/13 14:32:21]

These flags all look very similar to the ones in the .cc file. I'm confused how
this works now.

[slamm_google, 2012/04/13 23:43:36]

This is for a developer wanting to investigate one of the page loads. I looked
at calling performance_ui_test instead (which has a similar copy of options),
however, it runs the tests automatically, has a 10 minute time limit, and closes
the window as soon as the tests complete.

How about I add a comment to keep these in sync with performance_ui_tests? Not
ideal, but I cannot think of an easy way to have the list in one place.

[tonyg, 2012/04/16 15:34:32]

A comment is better than nothing. But it still seems like a shame to have two
harnesses as there's more duplication than just the flags.

Could we instead add new flags to performance_ui_test which allow it to be run
by a developer? Something that removes the timeout, suppresses closing the
window, etc?

If that doesn't work, then I don't see much benefit to adding it as a
performance_ui_test at all. Why not just have the buildbot call the python
script to run the test then?

(Please correct me if I'm missing something, just thinking aloud here)

[slamm_google, 2012/04/16 23:43:07]

I agree that there seems to be a fair amount of duplication with my current
approach. I originally chose to use performance_ui_test, to help make it easier
for the existing test maintainers to understand it. The duplication is certainly
counter productive for maintenance.

The value performance_ui_test adds that I can see:
  1. Tests all listed together.
  2. Adds a timeout (minor; webpagereplay_util.py would needs one)
  3. Closes the browser upon completion.
  4. Adds memory use and other perf stats (not sure how important they are).

I would like to get what I have running on the test machines before I invest
more time making it better. At this point, I do not even know if it will run and
give helpful results.

[tonyg, 2012/04/17 11:04:45]

Good point. It is important to get something running on the bots as an
incremental step. But then that only requires the performance_ui_tests version,
so perhaps this wrapper should be added in a subsequent patch?

[slamm_google, 2012/04/17 20:47:16]

runtest.py uses this module to start and stop WPR.

LaunchChromium() and main() are the extra parts for running without the bots.

+      '--disable-translate',
+      '--disable-background-networking',
+      '--enable-experimental-extension-apis',
+      '--enable-file-cookies',
+      '--enable-logging',
+      '--log-level=0',
+      '--enable-stats-table',
+      '--enable-benchmarking',
+      '--ignore-certificate-errors',
+      '--metrics-recording-only',
+      '--activate-on-launch',
+      '--no-first-run',
+      '--no-proxy-server',
+      '--user-data-dir=%s' % user_data_dir,
+      '--window-size=1280,1024',
+      ]
+  if not is_dns_forwarded:
+    chromium_args.append('--host-resolver-rules=MAP * %s' % REPLAY_HOST)
+  start_url = 'file://%s?test=%s' % (chromium_paths['start.html'], test_name)
+  if use_auto:
+    start_url += '&auto=1'
+  chromium_args.append(start_url)
+  if os.path.exists(user_data_dir):
+    shutil.rmtree(user_data_dir)
+  os.makedirs(user_data_dir)
+  try:
+    retval = subprocess.call(chromium_args)
+  finally:
+    shutil.rmtree(user_data_dir)
+
+
+def main():
+  log_level = logging.DEBUG
+  logging.basicConfig(level=log_level,
+                      format='%(asctime)s %(filename)s:%(lineno)-3d'
+                             ' %(levelname)s %(message)s',
+                      datefmt='%y%m%d %H:%M:%S')
+
+  option_parser = optparse.OptionParser(usage=USAGE)
+  option_parser.add_option(
+      '', '--auto', action='store_true', default=False,
+      help='Start test automatically.')
+  option_parser.add_option(
+      '', '--target', default='Release',
+      help='build target (Debug or Release)')
+  option_parser.add_option(
+      '', '--replay-dir', default=None,
+      help='Run replay from this directory instead of tools/build/third_party.')
+  replay_group = optparse.OptionGroup(option_parser,
+      'Options for replay.py', 'These options are passed through to replay.py.')
+  replay_group.add_option(
+      '', '--record', action='store_true', default=False,
+      help='Record a new WPR archive.')
+  replay_group.add_option(  # use default that does not require sudo
+      '', '--dns_forwarding', default=False, action='store_true',
+      help='Forward DNS requests to the local replay server.')
+  option_parser.add_option_group(replay_group)
+  options, args = option_parser.parse_args()
+  if len(args) != 1:
+    print >>sys.stderr, '%s: Error: Need exactly one TEST_NAME.' % sys.argv[0]
+    return 1
+  test_name = args[0]
+
+  chromium_paths = ChromiumPaths(
+      TARGET=options.target,
+      TEST_NAME=test_name,
+      TEST_EXE_NAME='webpagereplay_utils')
+  if os.path.exists(chromium_paths['archives']):
+    archive_path = chromium_paths['.wpr']
+  else:
+    archive_path = chromium_paths['.wpr_alt']
+  if not os.path.exists(archive_path) and not options.record:
+    print >>sys.stderr, 'Archive does not exist:', archive_path
+    return 1
+
+  replay_options = []
+  if options.record:
+    replay_options.append('--record')
+  if not options.dns_forwarding:
+    replay_options.append('--no-dns_forwarding')
+
+  if options.replay_dir:
+    replay_dir = options.replay_dir
+  else:
+    replay_dir = chromium_paths['replay']
+  wpr = ReplayLauncher(replay_dir, archive_path,
+                       chromium_paths['logs'], replay_options)
+  try:
+    wpr.StartServer()
+    LaunchChromium(chromium_paths, test_name,
+                   options.dns_forwarding, options.auto)
+  finally:
+    wpr.StopServer()
+  return 0
+
+if '__main__' == __name__:
+  sys.exit(main())