Add Web Page Replay test to page cycler.

tools/page_cycler/webpagereplay/extension/background.js

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// start.js sends a "start" message to set this.
+window.benchmarkConfiguration = {};
+
+// The callback (e.g. report writer) is set via AddBenchmarckCallback.
+window.benchmarkCallback;
+
+// Url to load before loading target page.
+var kWaitUrl = "http://wprwprwpr/web-page-replay-generate-200";

[tonyg, 2012/04/13 14:32:21]

This extension is very closely coupled with WPR. I think it should live in that
repo instead of here. What do you think?

[slamm_google, 2012/04/13 23:43:36]

The extension is what couples WPR and the page cycler, so it does not
necessarily fit in either place. I decided to put it here because I was thinking
of the WPR directory as server-side and the extension as client-side. The
client-side js is here too.

[tonyg, 2012/04/16 15:34:32]

OK, fair enough. I'm just sad to see duplication between this and the extension
in WPR. Need to check w/ simonjam, but maybe we should remove the extension in
WPR in favor of this one then.

[slamm_google, 2012/04/16 23:43:07]

Perhaps it could be made into a generic WPR extension if other extensions could
extract the data from it.

This version does work a little better than the perftracker extension now. The
perftracker checked for the page being done with an interval. I found that
method unreliable and it added a significant amount of time to the page loads
too.

[James Simonsen, 2012/04/17 03:30:19]

The only thing I really liked about the old one were the explicit cold, hot, and
warm test cases. I think there's 2 things that are missing:

1. Click back to get to exactly the same page. This is the hot cache case.

2. Start a new renderer, so we have no memory cache and read from the disk cache
exclusively. This is the warm cache case.

We get bits of #1 by navigating to another URL on the same host, which is good,
but I think the other cases are very common and important to measure. We can
incorporate them later though.

Otherwise, I don't think anyone will run the old extension, nor do we have any
reason to. WPR can focus on the server recording and replay aspect and we can
get rid of the extension and appspot code.

[tonyg, 2012/04/17 11:04:45]

Makes sense. I like the idea of porting those scenarios here and then removing
the extension from the WPR codebase. I also think it would make a lot of sense
to run these warm/hot test cases on the perfbots. Our behavior in  that regard
is completely in a blindspot right now.

+
+// Constant StatCounter Names
+var kTcpReadBytes = "tcp.read_bytes";
+var kTcpWriteBytes = "tcp.write_bytes";
+var kRequestCount = "HttpNetworkTransaction.Count";
+var kConnectCount = "tcp.connect";
+
+function CHECK(expr, comment) {
+  if (!expr) {
+    console.log(comment);
+    alert(comment);
+  }
+}
+
+// Return a list of values for an attribute (e.g. [x.attr for x in array]).
+Array.attrValues = function(array, attr, default_value) {
+  var values = [];
+  for (var i = 0, len = array.length; i < len; i++) {
+    if (attr in array[i]) {
+      values.push(array[i][attr]);
+    } else {
+      values.push(default_value);
+    }
+  }
+  return values;
+};
+
+// Returns the sum of all values in the array.
+Array.sum = function(array) {
+  var sum = 0;
+  for (var i = array.length - 1; i >= 0; i--) {
+    sum += array[i];
+  }
+  return sum;
+};
+
+// Map WebTiming properties to Result attribute names.

[tonyg, 2012/04/13 14:32:21]

Why bother with this map. Wouldn't it be more clear if we just reported the
navigation timing names?

[slamm_google, 2012/04/13 23:43:36]

I copied this from the perftracker code. A better comment would have been:

// Map Performance Timing names to Result attribute names.                      
// The Result values are the milliseconds from navigationStart.                 
// For example,                                                                 
//     var timing = window.performance.timing                                   
//     result['total_time'] = timing.loadEventEnd - timing.navigationStart;     

However, how about I drop the map as you suggest and special case a computation
of "total_time"? That is the only value used at the moment.

[tonyg, 2012/04/16 15:34:32]

Yeah, I like the idea of dropping the map.

The perf tests can report any number of traces. I think it would be incredibly
useful to report these times in addition to the total time. 

See here for an example of all the things that are tracked:
http://build.chromium.org/f/chromium/perf/xp-release-dual-core/moz/report.htm...

We could add the additional metrics to either the times tab or start a new tab
with the additional metrics.

[slamm_google, 2012/04/16 23:43:07]

The map is gone. The code is much easier to understand without it.

+window.loggedTimings = {
+  fetchStart: 'start_load_time',
+  domainLookupEnd: 'dns_time',
+  connectEnd: 'connect_time',
+  responseStart: 'first_byte_time',
+  responseEnd: 'last_byte_time',
+  domInteractive: 'doc_load_time',
+  domContentLoadedEventEnd: 'dcl_time',
+  loadEventEnd: 'total_time',
+};
+
+function Result() {
+  this.url = "";
+  this.paint_time = 0;
+  this.read_bytes_kb = 0;
+  this.write_bytes_kb = 0;
+  this.num_requests = 0;
+  this.num_connects = 0;
+  this.num_sessions = 0;
+  // The values of window.loggedTimings are also added as attributes.
+}
+
+// Collect all the results for a session (i.e. different pages).
+function ResultsCollection() {
+  var results_ = [];
+  var pages_ = [];
+  var pageResults_ = {};
+
+  this.addResult = function(result) {
+    results_.push(result);
+    var url = result.url;
+    if (!(url in pageResults_)) {
+      pages_.push(url);
+      pageResults_[url] = [];
+    }
+    pageResults_[url].push(result);
+  }
+
+  this.getPages = function() {
+    return pages_;
+  }
+
+  this.getResults = function() {
+    return results_;
+  }
+
+  this.getTotalTimes = function() {
+    var default_time = 0;
+    return Array.attrValues(results_, "total_time", default_time);
+  }
+}
+
+// Load a url in the default tab and record the time.
+function PageLoader(url, resultReadyCallback) {
+  var me_ = this;
+  var url_ = url;
+  var resultReadyCallback_ = resultReadyCallback;
+
+  // If it record mode, wait a little longer for lazy loaded resources.
+  var postLoadGraceMs_ = window.isRecordMode ? 5000 : 0;
+  var loadInterval_ = window.loadInterval;
+  var checkInterval_ = window.checkInterval;
+  var timeout_ = window.timeout;
+  var maxLoadChecks_ = window.maxLoadChecks;
+
+  var preloadFunc_;
+  var timeoutId_;
+  var isFinished_;
+  var result_;
+
+  var initialReadBytes_;
+  var initialWriteBytes_;
+  var initialRequestCount_;
+  var initialConnectCount_;
+
+  this.result = function() { return result_; };
+
+  this.run = function() {
+    timeoutId_ = null;
+    isFinished_ = false;
+    result_ = null;
+    initialReadBytes_ = chrome.benchmarking.counter(kTcpReadBytes);
+    initialWriteBytes_ = chrome.benchmarking.counter(kTcpWriteBytes);
+    initialRequestCount_ = chrome.benchmarking.counter(kRequestCount);
+    initialConnectCount_ = chrome.benchmarking.counter(kConnectCount);
+
+    if (me_.preloadFunc_) {
+      me_.preloadFunc_(me_.load_);
+    } else {
+      me_.load_();
+    }
+  };
+
+  this.setClearAll = function() {
+    me_.preloadFunc_ = me_.clearAll_;
+  };
+
+  this.setClearConnections = function() {
+    me_.preloadFunc_ = me_.clearConnections_;
+  };
+
+  this.clearAll_ = function(callback) {
+    chrome.tabs.getSelected(null, function(tab) {
+      chrome.tabs.update(tab.id, {"url": kWaitUrl});
+    });
+    var isDone = false;
+    chrome.browsingData.remove({}, {
+      "appcache": true,

[tonyg, 2012/04/13 14:32:21]

Is there a way to tell browsingData to remove everything? I'm wondering how
we'll remember to update this if things are added in the future.

[slamm_google, 2012/04/13 23:43:36]

I did not see a way. I even left a few things off the list: downloads,
fileSystems, formData, history, indexedDB, and passwords. One of them was giving
errors, so I tried taking off items that seemed less important -- though maybe I
should at least restore indexedDB.

http://code.google.com/chrome/extensions/trunk/browsingData.html

One way you could do it is by collecting all the chrome.browsingData.remove*
method names. Or, getting the owners to add a new method to clear everything.

[tonyg, 2012/04/16 15:34:32]

If there's no way to do them all, at least do as many as you can and leave a
comment about the error on the remaining one.

[slamm_google, 2012/04/16 23:43:07]

I added the full list again and they work fine -- I think I actually left some
off because I thought they were unnecessary. I also added code to add any new
remove* functions that show up.

+      "cache": true,
+      "cookies": true,
+      "localStorage": true,
+      "pluginData": true,
+      "webSQL": true
+    }, callback);
+    chrome.benchmarking.clearHostResolverCache();
+    chrome.benchmarking.clearPredictorCache();
+    chrome.benchmarking.closeConnections();
+  };
+
+  this.clearConnections_ = function(callback) {
+    chrome.benchmarking.closeConnections();
+    callback();
+  };
+
+  this.load_ = function() {
+    console.log("LOAD started: " + url_);
+    setTimeout(function() {
+      chrome.extension.onRequest.addListener(me_.finishLoad_);
+      timeoutId_ = setTimeout(function() {
+          me_.finishLoad_({"loadTimes": null, "timing": null});
+      }, timeout_);
+      chrome.tabs.getSelected(null, function(tab) {
+          chrome.tabs.update(tab.id, {"url": url_});
+      });
+    }, loadInterval_);
+  };
+
+  this.finishLoad_ = function(msg) {
+    if (!isFinished_) {
+      isFinished_ = true;
+      clearTimeout(timeoutId_);
+      chrome.extension.onRequest.removeListener(me_.finishLoad_);
+      me_.saveResult_(msg.loadTimes, msg.timing);
+    }
+  };
+
+  this.saveResult_ = function(loadTimes, timing) {
+    result_ = new Result()
+    result_.url = url_;
+    if (!loadTimes || !timing) {
+      console.log("LOAD INCOMPLETE: " + url_);
+    } else {
+      console.log("LOAD complete: " + url_);
+      var baseTime = timing.navigationStart;
+      CHECK(baseTime);
+      for (var prop in loggedTimings) {
+        if (prop in timing) {
+          result_[loggedTimings[prop]] = Math.max(0, timing[prop] - baseTime);
+        }
+      }
+      result_.paint_time = Math.max(0,
+          Math.round((1000.0 * loadTimes.firstPaintTime) - baseTime));
+    }
+    result_.read_bytes_kb = (chrome.benchmarking.counter(kTcpReadBytes) -
+                             initialReadBytes_) / 1024;
+    result_.write_bytes_kb = (chrome.benchmarking.counter(kTcpWriteBytes) -
+                              initialWriteBytes_) / 1024;
+    result_.num_requests = (chrome.benchmarking.counter(kRequestCount) -
+                            initialRequestCount_);
+    result_.num_connects = (chrome.benchmarking.counter(kConnectCount) -
+                            initialConnectCount_);
+    setTimeout(function() { resultReadyCallback_(me_); }, postLoadGraceMs_);
+  };
+}
+
+// Load page sets and prepare performance results.
+function SessionLoader(resultsReadyCallback) {
+  var me_ = this;
+  var resultsReadyCallback_ = resultsReadyCallback;
+  var pageSets_ = benchmarkConfiguration.pageSets;
+  var iterations_ = window.iterations;
+  var retries_ = window.retries;
+
+  var pageLoaders_ = [];
+  var resultsCollection_ = new ResultsCollection();
+  var loaderIndex_ = 0;
+  var retryIndex_ = 0;
+  var iterationIndex_ = 0;
+
+  this.run = function() {
+    me_.createLoaders_();
+    me_.loadPage_();
+  }
+
+  this.getResultsCollection = function() {
+    return resultsCollection_;
+  }
+
+  this.createLoaders_ = function() {
+    // Each url becomes one benchmark.
+    for (var i = 0; i < pageSets_.length; i++) {
+      for (var j = 0; j < pageSets_[i].length; j++) {
+        // Remove extra space at the beginning or end of a url.
+        var url = pageSets_[i][j].trim();
+        // Alert about and ignore blank page which does not get loaded.
+        if (url == "about:blank") {
+          alert("blank page loaded!");
+        } else if (!url.match(/https?:\/\//)) {
+          // Alert about url that is not in scheme http:// or https://.
+          alert("Skipping url without http:// or https://: " + url);
+        } else {
+          var loader = new PageLoader(url, me_.handleResult_)
+          if (j == 0) {
+            // Clear all browser data for the first page in a sub list.
+            loader.setClearAll();
+          } else {
+            // Otherwise, only clear the connections.
+            loader.setClearConnections();
+          }
+          pageLoaders_.push(loader);
+        }
+      }
+    }
+  }
+
+  this.loadPage_ = function() {
+    console.log("LOAD url " + (loaderIndex_ + 1) + " of " +
+                pageLoaders_.length +
+                ", iteration " + (iterationIndex_ + 1) + " of " +
+                iterations_);
+    pageLoaders_[loaderIndex_].run();
+  }
+
+  this.handleResult_ = function(loader) {
+    var result = loader.result();
+    resultsCollection_.addResult(result);
+    var totalTime = result["total_time"] || 0;
+    if (!totalTime && retryIndex_ < retries_) {
+      retryIndex_++;
+      console.log("LOAD retry, " + retryIndex_);
+    } else {
+      retryIndex_ = 0;
+      console.log("RESULTS url " + (loaderIndex_ + 1) + " of " +
+                  pageLoaders_.length +
+                  ", iteration " + (iterationIndex_ + 1) + " of " +
+                  iterations_ + ": " + result["total_time"]);
+      loaderIndex_++;
+      if (loaderIndex_ >= pageLoaders_.length) {
+        iterationIndex_++;
+        if (iterationIndex_ < iterations_) {
+          loaderIndex_ = 0;
+        } else {
+          resultsReadyCallback_(me_);
+          return;
+        }
+      }
+    }
+    me_.loadPage_();
+  }
+}
+
+function AddBenchmarkCallback(callback) {
+  window.benchmarkCallback = callback;
+}
+
+function Run() {
+  window.checkInterval = 500;
+  window.loadInterval = 1000;
+  window.timeout = 20000;  // max ms before killing page.
+  window.retries = 0;
+  window.isRecordMode = benchmarkConfiguration.isRecordMode;
+  if (window.isRecordMode) {
+    window.iterations = 1;
+    window.timeout = 40000;
+    window.retries = 2;
+  } else {
+    window.iterations = benchmarkConfiguration["iterations"] || 3;
+  }
+  var sessionLoader = new SessionLoader(benchmarkCallback);
+  console.log("pageSets: " + JSON.stringify(benchmarkConfiguration.pageSets));
+  sessionLoader.run();
+}
+
+chrome.extension.onConnect.addListener(function(port) {
+  port.onMessage.addListener(function(data) {
+    if (data.message == "start") {
+      window.benchmarkConfiguration = data.benchmark;
+      Run()
+    }
+  });
+});