Add Web Page Replay enabled page cycler tests to pyauto.

chrome/test/functional/webpagereplay.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A class to help start/stop a Web Page Replay Server.
 
-The page cycler tests use this module to run Web Page Replay
-(see tools/build/scripts/slave/runtest.py).
-
 If run from the command-line, the module will launch Web Page Replay
 and the specified test:
 
-   ./webpagereplay_utils.py --help  # list options
-   ./webpagereplay_utils.py 2012Q2  # run a WPR-enabled test
+   python webpagereplay.py --help  # list options
+   python webpagereplay.py CHROME_EXE 2012Q2  # run a WPR-enabled test
 """
 
 import logging
 import optparse
 import os
 import shutil
 import signal
 import subprocess
 import sys
 import tempfile
 import time
 import urllib
 
 
 USAGE = '%s [options] CHROME_EXE TEST_NAME' % os.path.basename(sys.argv[0])
 USER_DATA_DIR = '{TEMP}/webpagereplay_utils-chrome'
 
 # The port numbers must match those in chrome/test/perf/page_cycler_test.cc.
 HTTP_PORT = 8080
 HTTPS_PORT = 8413
+REPLAY_HOST='127.0.0.1'
 
 
 class ReplayError(Exception):
   """Catch-all exception for the module."""
   pass
 
 class ReplayNotFoundError(Exception):
   pass
 
 class ReplayNotStartedError(Exception):
   pass
 
 
-class ReplayLauncher(object):
+class ReplayServer(object):

[tonyg, 2012/05/18 15:37:54]

Probably worth a brief comment explaining how this is intended to be used (e.g.
the connection between __enter__/__exit__ and a with-block may not be obvious to
people less familiar w/ python.

[slamm_google, 2012/05/18 23:59:29]

Done.

   LOG_FILE = 'log.txt'
 
   def __init__(self, replay_dir, archive_path, log_dir, replay_options=None):
-    """Initialize ReplayLauncher.
+    """Initialize ReplayServer.
 
     Args:
       replay_dir: directory that has replay.py and related modules.
       archive_path: either a directory that contains WPR archives or,
           a path to a specific WPR archive.
       log_dir: where to write log.txt.
       replay_options: a list of options strings to forward to replay.py.
     """
     self.replay_dir = replay_dir
     self.archive_path = archive_path
@@ skipping 14 matching lines @@
         # TODO(slamm): Add traffic shaping (requires root):
         # '--net', 'fios',
         ]
     self.wpr_options.extend(self.replay_options)
 
   def _OpenLogFile(self):
     if not os.path.exists(self.log_dir):
       os.makedirs(self.log_dir)
     return open(self.log_name, 'w')
 
-  def StartServer(self):
-    cmd_line = [self.wpr_py]
-    cmd_line.extend(self.wpr_options)
-    # TODO(slamm): Support choosing archive on-the-fly.
-    cmd_line.append(self.archive_path)
-    self.log_fh = self._OpenLogFile()
-    logging.debug('Starting Web-Page-Replay: %s', cmd_line)
-    self.proxy_process = subprocess.Popen(
-      cmd_line, stdout=self.log_fh, stderr=subprocess.STDOUT)
-    if not self.IsStarted():
-      raise ReplayNotStartedError(
-          'Web Page Replay failed to start. See the log file: ' + self.log_name)
-
   def IsStarted(self):
     """Checks to see if the server is up and running."""
     for _ in range(5):
       if self.proxy_process.poll() is not None:
         # The process has exited.
         break
       try:
         up_url = '%s://localhost:%s/web-page-replay-generate-200'
         http_up_url = up_url % ('http', HTTP_PORT)
         https_up_url = up_url % ('https', HTTPS_PORT)
         if (200 == urllib.urlopen(http_up_url, None, {}).getcode() and
             200 == urllib.urlopen(https_up_url, None, {}).getcode()):
           return True
       except IOError:
         time.sleep(1)
     return False
 
-  def StopServer(self):
+  def __enter__(self):
+    cmd_line = [self.wpr_py]
+    cmd_line.extend(self.wpr_options)
+    # TODO(slamm): Support choosing archive on-the-fly.
+    cmd_line.append(self.archive_path)
+    self.log_fh = self._OpenLogFile()
+    logging.debug('Starting Web-Page-Replay: %s', cmd_line)
+    self.proxy_process = subprocess.Popen(
+      cmd_line, stdout=self.log_fh, stderr=subprocess.STDOUT)
+    if not self.IsStarted():
+      raise ReplayNotStartedError(
+          'Web Page Replay failed to start. See the log file: ' + self.log_name)
+
+  def __exit__(self, unused_exc_type, unused_exc_val, unused_exc_tb):
     if self.proxy_process:
       logging.debug('Stopping Web-Page-Replay')
       # Use a SIGINT here so that it can do graceful cleanup.
       # Otherwise, we will leave subprocesses hanging.
       self.proxy_process.send_signal(signal.SIGINT)
       self.proxy_process.wait()
     if self.log_fh:
       self.log_fh.close()
 
 
 class ChromiumPaths(object):
   """Collect all the path handling together."""
   PATHS = {
       'archives':   'src/data/page_cycler/webpagereplay',
       '.wpr':       'src/data/page_cycler/webpagereplay/{TEST_NAME}.wpr',
       '.wpr_alt':   'src/tools/page_cycler/webpagereplay/tests/{TEST_NAME}.wpr',
-      'start.html': 'src/tools/page_cycler/webpagereplay/start.html',
+      'start_url': ('src/tools/page_cycler/webpagereplay/start.html'
+                    '?test={TEST_NAME}'),
       'extension':  'src/tools/page_cycler/webpagereplay/extension',
       'replay':     'src/third_party/webpagereplay',
       'logs':       'src/webpagereplay_logs/{TEST_EXE_NAME}',
       }
 
   def __init__(self, **replacements):
     """Initialize ChromiumPaths.
 
     Args:
       replacements: a dict of format replacements for PATHS such as
           {'TEST_NAME': '2012Q2', 'TEST_EXE_NAME': 'performance_ui_tests'}.
     """
     module_dir = os.path.dirname(__file__)
     self.base_dir = os.path.abspath(os.path.join(
         module_dir, '..', '..', '..', '..'))
     self.replacements = replacements
 
+  def GetArchivePath(self):
+    if os.path.exists(self['archives']):
+      return self['.wpr']
+    return self['.wpr_alt']
+
+  def GetStartUrl(self, iterations=None, use_auto=False):
+    start_url = self['start_url']
+    if iterations is not None:
+      start_url += '&iterations=%d' % iterations
+    if use_auto:
+      start_url += '&auto=1'
+    return start_url
+
   def __getitem__(self, key):
     path_parts = [x.format(**self.replacements)
                   for x in self.PATHS[key].split('/')]
     return os.path.join(self.base_dir, *path_parts)
 
 
-def LaunchChromium(chrome_exe, chromium_paths, test_name,
+def ChromeFlags(existing_flags=None, extension_path=None,
+                is_dns_forwarded=False):
+  flags = []
+  if existing_flags:
+    flags.extend(existing_flags)
+  if extension_path:
+    flags.append('--load-extension=%s' % extension_path)
+  if not is_dns_forwarded:
+    flags.append('--host-resolver-rules=MAP * %s' % REPLAY_HOST)
+  flags.extend([
+      '--testing-fixed-http-port=%s' % HTTP_PORT,
+      '--testing-fixed-https-port=%s' % HTTPS_PORT,
+      '--log-level=0',
+      ])
+  extra_flags = [
+      '--disable-background-networking',
+      '--enable-experimental-extension-apis',
+      '--enable-file-cookies',

[tonyg, 2012/05/18 15:37:54]

This one isn't necessary since we aren't using file URLs, right?

[slamm_google, 2012/05/18 23:59:29]

The start URL is a file URL, and that is where the page cycler cookies are set.

+      '--enable-logging',
+      '--enable-stats-table',
+      '--enable-benchmarking',
+      '--ignore-certificate-errors',
+      '--metrics-recording-only',
+      '--activate-on-launch',
+      '--no-first-run',
+      '--no-proxy-server',
+      ]
+  flags.extend(f for f in extra_flags if f not in flags)
+  return flags
+
+
+def LaunchChrome(chrome_exe, chromium_paths, test_name,
                    is_dns_forwarded, use_auto):
-  """Launch chromium to run WPR-backed page cycler tests.
+  """Launch chrome to run WPR-backed page cycler tests.
 
   These options need to be kept in sync with
   src/chrome/test/perf/page_cycler_test.cc.
   """
-  REPLAY_HOST='127.0.0.1'
   user_data_dir = USER_DATA_DIR.format(**{'TEMP': tempfile.gettempdir()})
   chromium_args = [
       chrome_exe,
       '--load-extension=%s' % chromium_paths['extension'],
       '--testing-fixed-http-port=%s' % HTTP_PORT,
       '--testing-fixed-https-port=%s' % HTTPS_PORT,
       '--disable-background-networking',
       '--enable-experimental-extension-apis',
       '--enable-file-cookies',
       '--enable-logging',
       '--log-level=0',
       '--enable-stats-table',
       '--enable-benchmarking',
       '--ignore-certificate-errors',
       '--metrics-recording-only',
       '--activate-on-launch',
       '--no-first-run',
       '--no-proxy-server',

[slamm_google, 2012/05/18 00:20:24]

This list will go away in favor of the one above.

This function will need a few more edits -- was focusing on the perf test.

+
+
       '--user-data-dir=%s' % user_data_dir,
       '--window-size=1280,1024',
       ]
   if not is_dns_forwarded:
     chromium_args.append('--host-resolver-rules=MAP * %s' % REPLAY_HOST)
-  start_url = 'file://%s?test=%s' % (chromium_paths['start.html'], test_name)
-  if use_auto:
-    start_url += '&auto=1'
+  start_url = chromium_paths.GetStartUrl(test_name, use_auto=use_auto)
   chromium_args.append(start_url)
   if os.path.exists(user_data_dir):
     shutil.rmtree(user_data_dir)
   os.makedirs(user_data_dir)
   try:
     logging.debug('Starting Chrome: %s', chromium_args)
     retval = subprocess.call(chromium_args)
   finally:
     shutil.rmtree(user_data_dir)
 
 
 def main():

[tonyg, 2012/05/18 15:37:54]

Any reason not to just remove main entirely in this patch?

[slamm_google, 2012/05/18 23:59:29]

Ah, nice to toss this.

   log_level = logging.DEBUG
   logging.basicConfig(level=log_level,
                       format='%(asctime)s %(filename)s:%(lineno)-3d'
                              ' %(levelname)s %(message)s',
                       datefmt='%y%m%d %H:%M:%S')
 
   option_parser = optparse.OptionParser(usage=USAGE)
   option_parser.add_option(
       '', '--auto', action='store_true', default=False,
       help='Start test automatically.')
@@ skipping 15 matching lines @@
     return 1
   chrome_exe, test_name = args
 
   if not os.path.exists(chrome_exe):
     print >>sys.stderr, 'Chrome path does not exist:', chrome_exe
     return 1
 
   chromium_paths = ChromiumPaths(
       TEST_NAME=test_name,
       TEST_EXE_NAME='webpagereplay_utils')
-  if os.path.exists(chromium_paths['archives']):
-    archive_path = chromium_paths['.wpr']
-  else:
-    archive_path = chromium_paths['.wpr_alt']
+  archive_path = chromium_paths.GetArchivePath()
   if not os.path.exists(archive_path) and not options.record:
     print >>sys.stderr, 'Archive does not exist:', archive_path
     return 1
 
   replay_options = []
   if options.record:
     replay_options.append('--record')
   if not options.dns_forwarding:
     replay_options.append('--no-dns_forwarding')
 
   if options.replay_dir:
     replay_dir = options.replay_dir
   else:
     replay_dir = chromium_paths['replay']
-  wpr = ReplayLauncher(replay_dir, archive_path,
-                       chromium_paths['logs'], replay_options)
-  try:
-    wpr.StartServer()
-    LaunchChromium(chrome_exe, chromium_paths, test_name,
-                   options.dns_forwarding, options.auto)
-  finally:
-    wpr.StopServer()
+  with ReplayServer(
+      replay_dir, archive_path, chromium_paths['logs'], replay_options):
+    LaunchChrome(chrome_exe, chromium_paths, test_name,
+                 options.dns_forwarding, options.auto)
   return 0
 
 if '__main__' == __name__:
   sys.exit(main())