Add a copy of the Kraken Benchmark for performance testing.

chrome/test/data/third_party/kraken/hosted/compare-results.js

+/*
+ * Copyright (C) 2007 Apple Inc.  All rights reserved.
+ * Copyright (C) 2010 Mozilla Foundation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+function sunspiderCompareResults(output1, output2)
+{
+    var count1 = output1.length;
+    var count2 = output2.length;
+
+    var itemTotals1 = {};
+    itemTotals1.length = count1;
+    
+    var total1 = 0;
+    var categoryTotals1 = {};
+    var testTotalsByCategory1 = {};
+    
+    var mean1 = 0;
+    var categoryMeans1 = {};
+    var testMeansByCategory1 = {};
+    
+    var stdDev1 = 0;
+    var categoryStdDevs1 = {};
+    var testStdDevsByCategory1 = {};
+    
+    var stdErr1 = 0;
+    var categoryStdErrs1 = {};
+    var testStdErrsByCategory1 = {};
+    
+    var itemTotals2 = {};
+    itemTotals2.length = count2;
+    
+    var total2 = 0;
+    var categoryTotals2 = {};
+    var testTotalsByCategory2 = {};
+    
+    var mean2 = 0;
+    var categoryMeans2 = {};
+    var testMeansByCategory2 = {};
+    
+    var stdDev2 = 0;
+    var categoryStdDevs2 = {};
+    var testStdDevsByCategory2 = {};
+    
+    var stdErr2 = 0;
+    var categoryStdErrs2 = {};
+    var testStdErrsByCategory2 = {};
+    
+    function initialize()
+    {
+        itemTotals1 = {total: []};
+        
+        for (var i = 0; i < categories.length; i++) {
+            var category = categories[i];
+            itemTotals1[category] = [];
+            categoryTotals1[category] = 0;
+            testTotalsByCategory1[category] = {};
+            categoryMeans1[category] = 0;
+            testMeansByCategory1[category] = {};
+            categoryStdDevs1[category] = 0;
+            testStdDevsByCategory1[category] = {};
+            categoryStdErrs1[category] = 0;
+            testStdErrsByCategory1[category] = {};
+        }
+        
+        for (var i = 0; i < tests.length; i++) {
+            var test = tests[i];
+            itemTotals1[test] = [];
+            var category = test.replace(/-.*/, "");
+            testTotalsByCategory1[category][test] = 0;
+            testMeansByCategory1[category][test] = 0;
+            testStdDevsByCategory1[category][test] = 0;
+            testStdErrsByCategory1[category][test] = 0;
+        }
+        
+        for (var i = 0; i < count1; i++) {
+            itemTotals1["total"][i] = 0;
+            for (var category in categoryTotals1) {
+                itemTotals1[category][i] = 0;
+                for (var test in testTotalsByCategory1[category]) {
+                    itemTotals1[test][i] = 0;
+                }
+            }
+        }
+        
+        itemTotals2 = {total: []};
+        
+        for (var i = 0; i < categories.length; i++) {
+            var category = categories[i];
+            itemTotals2[category] = [];
+            categoryTotals2[category] = 0;
+            testTotalsByCategory2[category] = {};
+            categoryMeans2[category] = 0;
+            testMeansByCategory2[category] = {};
+            categoryStdDevs2[category] = 0;
+            testStdDevsByCategory2[category] = {};
+            categoryStdErrs2[category] = 0;
+            testStdErrsByCategory2[category] = {};
+        }
+        
+        for (var i = 0; i < tests.length; i++) {
+            var test = tests[i];
+            itemTotals2[test] = [];
+            var category = test.replace(/-.*/, "");
+            testTotalsByCategory2[category][test] = 0;
+            testMeansByCategory2[category][test] = 0;
+            testStdDevsByCategory2[category][test] = 0;
+            testStdErrsByCategory2[category][test] = 0;
+        }
+        
+        for (var i = 0; i < count2; i++) {
+            itemTotals2["total"][i] = 0;
+            for (var category in categoryTotals2) {
+                itemTotals2[category][i] = 0;
+                for (var test in testTotalsByCategory2[category]) {
+                    itemTotals2[test][i] = 0;
+                }
+            }
+        }
+        
+    }
+    
+    function computeItemTotals(output, itemTotals)
+    {
+        for (var i = 0; i < output.length; i++) {
+            var result = output[i];
+            for (var test in result) {
+                var time = result[test];
+                var category = test.replace(/-.*/, "");
+                itemTotals["total"][i] += time;
+                itemTotals[category][i] += time;
+                itemTotals[test][i] += time;
+            }
+        }
+    }
+    
+    function computeTotals(output, categoryTotals, testTotalsByCategory)
+    {
+        var total = 0;
+        
+        for (var i = 0; i < output.length; i++) {
+            var result = output[i];
+            for (var test in result) {
+                var time = result[test];
+                var category = test.replace(/-.*/, "");
+                total += time;
+                categoryTotals[category] += time;
+                testTotalsByCategory[category][test] += time;
+            }
+        }
+        
+        return total;
+    }
+    
+    function computeMeans(count, total, categoryTotals, categoryMeans, testTotalsByCategory, testMeansByCategory)
+    {
+        var mean = total / count;
+        for (var category in categoryTotals) {
+            categoryMeans[category] = categoryTotals[category] / count;
+            for (var test in testTotalsByCategory[category]) {
+                testMeansByCategory[category][test] = testTotalsByCategory[category][test] / count;
+            }
+        }
+        return mean;
+    }
+    
+    function standardDeviation(mean, items)
+    {
+        var deltaSquaredSum = 0;
+        for (var i = 0; i < items.length; i++) {
+            var delta = items[i] - mean;
+            deltaSquaredSum += delta * delta;
+        }
+        variance = deltaSquaredSum / (items.length - 1);
+        return Math.sqrt(variance);
+    }
+    
+    function computeStdDevs(mean, itemTotals, categoryStdDevs, categoryMeans, testStdDevsByCategory, testMeansByCategory)
+    {
+        var stdDev = standardDeviation(mean, itemTotals["total"]);
+        for (var category in categoryStdDevs) {
+            categoryStdDevs[category] = standardDeviation(categoryMeans[category], itemTotals[category]);
+        }
+        for (var category in categoryStdDevs) {
+            for (var test in testStdDevsByCategory[category]) {
+                testStdDevsByCategory[category][test] = standardDeviation(testMeansByCategory[category][test], itemTotals[test]);
+            }
+        }
+        return stdDev;
+    }
+    
+    function computeStdErrors(count, stdDev, categoryStdErrs, categoryStdDevs, testStdErrsByCategory, testStdDevsByCategory)
+    {
+        var sqrtCount = Math.sqrt(count);
+        
+        var stdErr = stdDev / sqrtCount;
+        for (var category in categoryStdErrs) {
+            categoryStdErrs[category] = categoryStdDevs[category] / sqrtCount;
+        }
+        for (var category in categoryStdDevs) {
+            for (var test in testStdErrsByCategory[category]) {
+                testStdErrsByCategory[category][test] = testStdDevsByCategory[category][test] / sqrtCount;
+            }
+        }
+        
+        return stdErr;
+    }
+    
+    var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
+    var tMax = tDistribution.length;
+    var tLimit = 1.96;
+    
+    function tDist(n)
+    {
+        if (n > tMax)
+            return tLimit;
+        return tDistribution[n];
+    }
+    
+    
+    function formatMean(meanWidth, mean, stdErr, count)
+    {
+        var meanString = mean.toFixed(1).toString();
+        while (meanString.length < meanWidth) {
+            meanString = " " + meanString;
+        }
+        
+        var error = "+/- " + ((tDist(count) * stdErr / mean) * 100).toFixed(1) + "% ";
+        
+        return meanString + "ms " + error;
+    }
+    
+    function computeLabelWidth()
+    {
+        var width = "Total".length;
+        for (var category in categoryMeans1) {
+            if (category.length + 2 > width)
+                width = category.length + 2;
+        }
+        for (var i = 0; i < tests.length; i++) {
+            var shortName = tests[i].replace(/^[^-]*-/, "");
+            if (shortName.length + 4 > width)
+                width = shortName.length + 4;
+        }
+        
+        return width;
+    }
+    
+    function computeMeanWidth(mean, categoryMeans, testMeansByCategory)
+    {
+        var width = mean.toFixed(1).toString().length;
+        for (var category in categoryMeans) {
+            var candidate = categoryMeans[category].toFixed(1).toString().length;
+            if (candidate > width)
+                width = candidate;
+            for (var test in testMeansByCategory[category]) {
+                var candidate = testMeansByCategory[category][test].toFixed(1).toString().length;
+                if (candidate > width)
+                    width = candidate;
+            }
+        }
+        
+        return width;
+    }
+    
+    function pad(str, n)
+    {
+        while (str.length < n) {
+            str += " ";
+        }
+        return str;
+    }
+    
+    function resultLine(labelWidth, indent, label, meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2)
+    {
+        result = pad("", indent);    
+        result += label + ": ";
+        result = pad(result, labelWidth + 2);
+        
+        var t = (mean1 - mean2) / (Math.sqrt((stdErr1 * stdErr1) + (stdErr2 * stdErr2)));
+        var df = count1 + count2 - 2;
+        
+        var statisticallySignificant = (Math.abs(t) > tDist(df+1));
+        var diff = mean2 - mean1;
+        var percentage = 100 * diff / mean1;
+        var isFaster = diff < 0;
+        var probablySame = (percentage < 0.1) && !statisticallySignificant;
+        var ratio = isFaster ? (mean1 / mean2) : (mean2 / mean1);
+        var fixedRatio = (ratio < 1.2) ? ratio.toFixed(3).toString() : ((ratio < 10) ? ratio.toFixed(2).toString() : ratio.toFixed(1).toString());
+        var formattedRatio = isFaster ? fixedRatio + "x as fast" : "*" + fixedRatio + "x as slow*";
+
+        var diffSummary;
+        var diffDetail;
+        
+        if (probablySame) {
+            diffSummary = "-";
+            diffDetail = "";
+        } else if (!statisticallySignificant) {
+            diffSummary = "??";
+            diffDetail =  "    might be " + formattedRatio;
+        } else {
+            diffSummary = formattedRatio;
+            diffDetail = "    significant"; 
+        }
+        
+        return result + pad(diffSummary, 18) + formatMean(meanWidth1, mean1, stdErr1, count1) + "  " + formatMean(meanWidth2, mean2, stdErr2, count2) + diffDetail;
+    }
+    
+    function printOutput()
+    {
+        var labelWidth = computeLabelWidth();
+        var meanWidth1 = computeMeanWidth(mean1, categoryMeans1, testMeansByCategory1);
+        var meanWidth2 = computeMeanWidth(mean2, categoryMeans2, testMeansByCategory2);
+        
+        print("\n");
+        var header = "TEST";
+        while (header.length < labelWidth)
+            header += " ";
+        header += "  COMPARISON            FROM                 TO               DETAILS";
+        print(header);
+        print("");
+        print("====================================================================================");
+        print("");
+        print(resultLine(labelWidth, 0, "** TOTAL **", meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2));
+        print("");
+        print("====================================================================================");
+        
+        for (var category in categoryMeans1) {
+            print("");
+            print(resultLine(labelWidth, 2, category,
+                             meanWidth1, categoryMeans1[category], categoryStdErrs1[category],
+                             meanWidth2, categoryMeans2[category], categoryStdErrs2[category]));
+            for (var test in testMeansByCategory1[category]) {
+                var shortName = test.replace(/^[^-]*-/, "");
+                print(resultLine(labelWidth, 4, shortName, 
+                                 meanWidth1, testMeansByCategory1[category][test], testStdErrsByCategory1[category][test],
+                                 meanWidth2, testMeansByCategory2[category][test], testStdErrsByCategory2[category][test]));
+            }
+        }
+    }
+    
+    initialize();
+    
+    computeItemTotals(output1, itemTotals1);
+    computeItemTotals(output2, itemTotals2);
+
+    total1 = computeTotals(output1, categoryTotals1, testTotalsByCategory1);
+    total2 = computeTotals(output2, categoryTotals2, testTotalsByCategory2);
+
+    mean1 = computeMeans(count1, total1, categoryTotals1, categoryMeans1, testTotalsByCategory1, testMeansByCategory1);
+    mean2 = computeMeans(count2, total2, categoryTotals2, categoryMeans2, testTotalsByCategory2, testMeansByCategory2);
+
+    stdDev1 = computeStdDevs(mean1, itemTotals1, categoryStdDevs1, categoryMeans1, testStdDevsByCategory1, testMeansByCategory1);
+    stdDev2 = computeStdDevs(mean2, itemTotals2, categoryStdDevs2, categoryMeans2, testStdDevsByCategory2, testMeansByCategory2);
+
+    stdErr1 = computeStdErrors(count1, stdDev1, categoryStdErrs1, categoryStdDevs1, testStdErrsByCategory1, testStdDevsByCategory1);
+    stdErr2 = computeStdErrors(count2, stdDev2, categoryStdErrs2, categoryStdDevs2, testStdErrsByCategory2, testStdDevsByCategory2);
+
+    printOutput();
+}