Setting up benchmarks and perf utilities

test/perf/compare.dart

+#!/usr/bin/env dart
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/**
+ * Script that prints in a table format a comparison of two benchmark results.
+ * The input is given as two json files containing the benchmark results.
+ */

[Jennifer Messerly, 2013/01/07 21:07:46]

library test.perf.compare; ?

[Siggi Cherem (dart-lang), 2013/01/08 02:26:18]

Done.

+
+import 'dart:io';
+import 'dart:json';
+import 'dart:math' as math;
+
+main() {
+  var args = new Options().arguments;
+  if (args.length < 2) {
+    print('usage: compare.dart results1.json results2.json [filter]');
+    exit(1);
+  }
+
+  var path1 = args[0];
+  var path2 = args[1];
+  var file1 = new File(path1).readAsStringSync();
+  var file2 = new File(path2).readAsStringSync();
+  var filter = args.length > 2 ? new RegExp(args[2]) : null;
+
+  var results = [];
+  var map1 = JSON.parse(file1);
+  var map2 = JSON.parse(file2);
+
+  for (var key in map1.keys) {
+    if (map2.containsKey(key)) {
+      results.add(new _ResultPair(key, map1[key], map2[key]));
+    } else {
+      results.add(new _ResultPair(key, map1[key], null));
+    }
+  }
+
+  for (var key in map2.keys) {
+    if (!map1.containsKey(key)) {
+      results.add(new _ResultPair(key, null, map2[key]));
+    }
+  }
+
+  print('Comparing:\n  (a) $path1\n  (b) $path2');
+  _printLine('Benchmark', '(a)', '(b)', '% (b - a)/a');
+  _printLine('------------------------------', '--------------',
+        '--------------', '--------------');
+  var scores1 = [];
+  var scores2 = [];
+  var someNull = false;
+  if (filter != null) {
+    results = results.filter((s) => filter.hasMatch(s.name));
+  }
+  results.sort((a, b) {
+    if (a.name.startsWith('dart') && b.name.startsWith('js  ')) return 1;

[Jennifer Messerly, 2013/01/07 21:07:46]

since ResultPair is its own class, perhaps make it implemented Comparable and
move this logic into the compareTo method?

[Siggi Cherem (dart-lang), 2013/01/08 02:26:18]

Done.

+    if (a.name.startsWith('js  ') && b.name.startsWith('dart')) return -1;
+    if (a.factor != null && b.factor != null) {
+      var res = a.factor.compareTo(b.factor);
+      if (res != 0) return res;
+    }
+    return a.name.compareTo(b.name);
+  });
+
+  for (var entry in results) {
+    print(entry);
+    if (entry.score1 != null) {
+      scores1.add(entry.time1);
+    } else {
+      someNull = true;
+    }
+    if (entry.score2 != null) {
+      scores2.add(entry.time2);
+    } else {
+      someNull = true;
+    }
+
+  }
+  _printLine('------------------------------', '--------------',
+        '--------------', '--------------');
+  print(new _ResultPair('Geometric mean', _geomean(scores1),
+        _geomean(scores2)));
+
+  if (someNull) {
+    scores1 = [];
+    scores2 = [];
+    for (var entry in results) {
+      if (entry.score1 != null && entry.score2 !=  null) {
+        scores1.add(entry.score1);
+        scores2.add(entry.score2);
+      }
+    }
+    print(new _ResultPair('Geometric mean (both avail)',
+          _geomean(scores1), _geomean(scores2)));
+  }
+}
+
+class _ResultPair {

[Jennifer Messerly, 2013/01/07 21:07:46]

general note: I don't think most of these classes really needed to be private,
since this is an entry point it's not likely to be imported. (Actually since it
doesn't have a library tag, it can't be imported)

Not a big deal either way though.

[Siggi Cherem (dart-lang), 2013/01/08 02:26:18]

Done.

+  final String name;
+  final num time1;
+  final num time2;
+  final num score1;
+  final num score2;
+
+  num get factor => score1 == null || score2 == null ? null
+      : ((score2 - score1) * 100.0) / score1;
+
+  _ResultPair(this.name, double time1, double time2)
+      : this.time1 = time1, this.time2 = time2,
+        score1 = time1 == null ? null : 1000000.0 / time1,
+        score2 = time2 == null ? null : 1000000.0 / time2;
+
+  String toString() {
+    var buff = new StringBuffer();
+    buff.add(name);
+    _ensure(buff, 30);
+    _addNumber(buff, score1, 45);
+    _addNumber(buff, score2, 60);
+    _addNumber(buff, factor, 75, color: true);
+    return buff.toString();
+  }
+}
+
+_printLine(String col0, String col1, String col2, String col3) {
+  var buff = new StringBuffer();
+  buff.add(col0);
+  _ensure(buff, 30);
+  _padRight(buff, col1, 45);
+  _padRight(buff, col2, 60);
+  _padRight(buff, col3, 75);
+  print(buff.toString());
+}
+
+_ensure(StringBuffer buff, int ensure) {

[Jennifer Messerly, 2013/01/07 21:07:46]

I wonder if something like _padSpaces or _ensureColumn would be a better name?

[Siggi Cherem (dart-lang), 2013/01/08 02:26:18]

Done.

+  while (buff.length < ensure) {
+    buff.add(' ');
+  }
+}
+
+_addNumber(StringBuffer buff, num value, int ensure, {bool color: false}) {
+  var str;
+  if (value == null) {
+    str = '--';
+  } else {
+    str = (value > 100) ? value.toStringAsFixed(0) : value.toStringAsFixed(2);

[Jennifer Messerly, 2013/01/07 21:07:46]

If value is between 10 and 100 should it use toStringAsFixed(1)?

Should it be value >= 100, since 100 is already 3 digits?

[Siggi Cherem (dart-lang), 2013/01/08 02:26:18]

Done.

+  }
+  while (buff.length + str.length < ensure) {
+    buff.add(' ');
+  }
+  if (color) _addColor(buff, value);
+  buff.add(str);
+  if (color) _removeColor(buff, value);
+}
+
+_addColor(StringBuffer buff, num value) {
+  if (value == null || value.abs() < 2) return;
+  var color;
+  if (value >= 2 && value < 7) {
+    color = '';
+  } else if (value >= 7) {
+    color = '';
+  } else if (value <= -2 && value > -7) {
+    color = '';
+  } else if (value <= -7) {
+    color = '';
+  }
+  buff.add(color);
+}
+
+_removeColor(StringBuffer buff, num value) {
+  if (value == null || value.abs() < 2) return;
+  buff.add('');
+}
+
+_padRight(StringBuffer buff, String str, int ensure) {
+  while (buff.length + str.length < ensure) {
+    buff.add(' ');
+  }
+  buff.add(str);
+}
+
+_geomean(List<num> numbers) {
+  var log = 0.0;
+  for (var n in numbers) {
+    log += math.log(n);
+  }
+  return math.pow(math.E, log / numbers.length);
+}