OLD | NEW |
| (Empty) |
1 // Copyright 2008 the V8 project authors. All rights reserved. | |
2 // Redistribution and use in source and binary forms, with or without | |
3 // modification, are permitted provided that the following conditions are | |
4 // met: | |
5 // | |
6 // * Redistributions of source code must retain the above copyright | |
7 // notice, this list of conditions and the following disclaimer. | |
8 // * Redistributions in binary form must reproduce the above | |
9 // copyright notice, this list of conditions and the following | |
10 // disclaimer in the documentation and/or other materials provided | |
11 // with the distribution. | |
12 // * Neither the name of Google Inc. nor the names of its | |
13 // contributors may be used to endorse or promote products derived | |
14 // from this software without specific prior written permission. | |
15 // | |
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
17 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
18 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
19 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
20 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
21 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
22 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
23 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
24 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
25 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
26 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
27 | |
28 | |
29 // Simple framework for running the benchmark suites and | |
30 // computing a score based on the timing measurements. | |
31 | |
32 | |
33 // A benchmark has a name (string) and a function that will be run to | |
34 // do the performance measurement. The optional setup and tearDown | |
35 // arguments are functions that will be invoked before and after | |
36 // running the benchmark, but the running time of these functions will | |
37 // not be accounted for in the benchmark score. | |
38 function Benchmark(name, run, setup, tearDown) { | |
39 this.name = name; | |
40 this.run = run; | |
41 this.Setup = setup ? setup : function() { }; | |
42 this.TearDown = tearDown ? tearDown : function() { }; | |
43 } | |
44 | |
45 | |
46 // Benchmark results hold the benchmark and the measured time used to | |
47 // run the benchmark. The benchmark score is computed later once a | |
48 // full benchmark suite has run to completion. | |
49 function BenchmarkResult(benchmark, time) { | |
50 this.benchmark = benchmark; | |
51 this.time = time; | |
52 } | |
53 | |
54 | |
55 // Automatically convert results to numbers. Used by the geometric | |
56 // mean computation. | |
57 BenchmarkResult.prototype.valueOf = function() { | |
58 return this.time; | |
59 } | |
60 | |
61 | |
62 // Suites of benchmarks consist of a name and the set of benchmarks in | |
63 // addition to the reference timing that the final score will be based | |
64 // on. This way, all scores are relative to a reference run and higher | |
65 // scores implies better performance. | |
66 function BenchmarkSuite(name, reference, benchmarks) { | |
67 this.name = name; | |
68 this.reference = reference; | |
69 this.benchmarks = benchmarks; | |
70 BenchmarkSuite.suites.push(this); | |
71 } | |
72 | |
73 | |
74 // Keep track of all declared benchmark suites. | |
75 BenchmarkSuite.suites = []; | |
76 | |
77 | |
78 // Scores are not comparable across versions. Bump the version if | |
79 // you're making changes that will affect that scores, e.g. if you add | |
80 // a new benchmark or change an existing one. | |
81 BenchmarkSuite.version = '6'; | |
82 | |
83 | |
84 // To make the benchmark results predictable, we replace Math.random | |
85 // with a 100% deterministic alternative. | |
86 Math.random = (function() { | |
87 var seed = 49734321; | |
88 return function() { | |
89 // Robert Jenkins' 32 bit integer hash function. | |
90 seed = ((seed + 0x7ed55d16) + (seed << 12)) & 0xffffffff; | |
91 seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff; | |
92 seed = ((seed + 0x165667b1) + (seed << 5)) & 0xffffffff; | |
93 seed = ((seed + 0xd3a2646c) ^ (seed << 9)) & 0xffffffff; | |
94 seed = ((seed + 0xfd7046c5) + (seed << 3)) & 0xffffffff; | |
95 seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff; | |
96 return (seed & 0xfffffff) / 0x10000000; | |
97 }; | |
98 })(); | |
99 | |
100 | |
101 // Runs all registered benchmark suites and optionally yields between | |
102 // each individual benchmark to avoid running for too long in the | |
103 // context of browsers. Once done, the final score is reported to the | |
104 // runner. | |
105 BenchmarkSuite.RunSuites = function(runner) { | |
106 var continuation = null; | |
107 var suites = BenchmarkSuite.suites; | |
108 var length = suites.length; | |
109 BenchmarkSuite.scores = []; | |
110 var index = 0; | |
111 function RunStep() { | |
112 while (continuation || index < length) { | |
113 if (continuation) { | |
114 continuation = continuation(); | |
115 } else { | |
116 var suite = suites[index++]; | |
117 if (runner.NotifyStart) runner.NotifyStart(suite.name); | |
118 continuation = suite.RunStep(runner); | |
119 } | |
120 if (continuation && typeof window != 'undefined' && window.setTimeout) { | |
121 window.setTimeout(RunStep, 25); | |
122 return; | |
123 } | |
124 } | |
125 if (runner.NotifyScore) { | |
126 var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores); | |
127 var formatted = BenchmarkSuite.FormatScore(100 * score); | |
128 runner.NotifyScore(formatted); | |
129 } | |
130 } | |
131 RunStep(); | |
132 } | |
133 | |
134 | |
135 // Counts the total number of registered benchmarks. Useful for | |
136 // showing progress as a percentage. | |
137 BenchmarkSuite.CountBenchmarks = function() { | |
138 var result = 0; | |
139 var suites = BenchmarkSuite.suites; | |
140 for (var i = 0; i < suites.length; i++) { | |
141 result += suites[i].benchmarks.length; | |
142 } | |
143 return result; | |
144 } | |
145 | |
146 | |
147 // Computes the geometric mean of a set of numbers. | |
148 BenchmarkSuite.GeometricMean = function(numbers) { | |
149 var log = 0; | |
150 for (var i = 0; i < numbers.length; i++) { | |
151 log += Math.log(numbers[i]); | |
152 } | |
153 return Math.pow(Math.E, log / numbers.length); | |
154 } | |
155 | |
156 | |
157 // Converts a score value to a string with at least three significant | |
158 // digits. | |
159 BenchmarkSuite.FormatScore = function(value) { | |
160 if (value > 100) { | |
161 return value.toFixed(0); | |
162 } else { | |
163 return value.toPrecision(3); | |
164 } | |
165 } | |
166 | |
167 // Notifies the runner that we're done running a single benchmark in | |
168 // the benchmark suite. This can be useful to report progress. | |
169 BenchmarkSuite.prototype.NotifyStep = function(result) { | |
170 this.results.push(result); | |
171 if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name); | |
172 } | |
173 | |
174 | |
175 // Notifies the runner that we're done with running a suite and that | |
176 // we have a result which can be reported to the user if needed. | |
177 BenchmarkSuite.prototype.NotifyResult = function() { | |
178 var mean = BenchmarkSuite.GeometricMean(this.results); | |
179 var score = this.reference / mean; | |
180 BenchmarkSuite.scores.push(score); | |
181 if (this.runner.NotifyResult) { | |
182 var formatted = BenchmarkSuite.FormatScore(100 * score); | |
183 this.runner.NotifyResult(this.name, formatted); | |
184 } | |
185 } | |
186 | |
187 | |
188 // Notifies the runner that running a benchmark resulted in an error. | |
189 BenchmarkSuite.prototype.NotifyError = function(error) { | |
190 if (this.runner.NotifyError) { | |
191 this.runner.NotifyError(this.name, error); | |
192 } | |
193 if (this.runner.NotifyStep) { | |
194 this.runner.NotifyStep(this.name); | |
195 } | |
196 } | |
197 | |
198 | |
199 // Runs a single benchmark for at least a second and computes the | |
200 // average time it takes to run a single iteration. | |
201 BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) { | |
202 function Measure(data) { | |
203 var elapsed = 0; | |
204 var start = new Date(); | |
205 for (var n = 0; elapsed < 1000; n++) { | |
206 benchmark.run(); | |
207 elapsed = new Date() - start; | |
208 } | |
209 if (data != null) { | |
210 data.runs += n; | |
211 data.elapsed += elapsed; | |
212 } | |
213 } | |
214 | |
215 if (data == null) { | |
216 // Measure the benchmark once for warm up and throw the result | |
217 // away. Return a fresh data object. | |
218 Measure(null); | |
219 return { runs: 0, elapsed: 0 }; | |
220 } else { | |
221 Measure(data); | |
222 // If we've run too few iterations, we continue for another second. | |
223 if (data.runs < 32) return data; | |
224 var usec = (data.elapsed * 1000) / data.runs; | |
225 this.NotifyStep(new BenchmarkResult(benchmark, usec)); | |
226 return null; | |
227 } | |
228 } | |
229 | |
230 | |
231 // This function starts running a suite, but stops between each | |
232 // individual benchmark in the suite and returns a continuation | |
233 // function which can be invoked to run the next benchmark. Once the | |
234 // last benchmark has been executed, null is returned. | |
235 BenchmarkSuite.prototype.RunStep = function(runner) { | |
236 this.results = []; | |
237 this.runner = runner; | |
238 var length = this.benchmarks.length; | |
239 var index = 0; | |
240 var suite = this; | |
241 var data; | |
242 | |
243 // Run the setup, the actual benchmark, and the tear down in three | |
244 // separate steps to allow the framework to yield between any of the | |
245 // steps. | |
246 | |
247 function RunNextSetup() { | |
248 if (index < length) { | |
249 try { | |
250 suite.benchmarks[index].Setup(); | |
251 } catch (e) { | |
252 suite.NotifyError(e); | |
253 return null; | |
254 } | |
255 return RunNextBenchmark; | |
256 } | |
257 suite.NotifyResult(); | |
258 return null; | |
259 } | |
260 | |
261 function RunNextBenchmark() { | |
262 try { | |
263 data = suite.RunSingleBenchmark(suite.benchmarks[index], data); | |
264 } catch (e) { | |
265 suite.NotifyError(e); | |
266 return null; | |
267 } | |
268 // If data is null, we're done with this benchmark. | |
269 return (data == null) ? RunNextTearDown : RunNextBenchmark(); | |
270 } | |
271 | |
272 function RunNextTearDown() { | |
273 try { | |
274 suite.benchmarks[index++].TearDown(); | |
275 } catch (e) { | |
276 suite.NotifyError(e); | |
277 return null; | |
278 } | |
279 return RunNextSetup; | |
280 } | |
281 | |
282 // Start out running the setup. | |
283 return RunNextSetup(); | |
284 } | |
OLD | NEW |