Support for pixel layout tests.

utils/testrunner/layout_test_controller.dart

Index: utils/testrunner/layout_test_controller.dart
===================================================================
--- utils/testrunner/layout_test_controller.dart	(revision 0)
+++ utils/testrunner/layout_test_controller.dart	(revision 0)
@@ -0,0 +1,279 @@
+// The following set of variables should be set by the caller.

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

several comments here:
- add copyright notice header
- add #library?
- add general library comment (e.g. this is a runtime library used by code
generated by 'testrunner' to run layout tests)
- clarify "set by the caller": how? when? the caller of what?

[gram, 2012/09/20 18:58:03]

This is #sourced by the generated Dart file, hence no #directives.

+/** Whether to include elapsed time. */
+bool includeTime;
+
+/** Path to DRT executable. */
+String drt;
+
+/** Whether to regenerate layout test files. */
+bool regenerate;
+
+/** Whether to output test summary. */
+bool summarize;
+
+/** Format strings to use for test result messages. */
+String passFormat, failFormat, errorFormat, listFormat;
+
+/** Location of the running test file. */
+String sourceDir;
+
+/** Path of the running test file. */
+String testfile;
+
+/** URL of the child test file. */
+String baseUrl;
+
+// Variable below here are local to this file.
+var passCount = 0, failCount = 0, errorCount = 0;
+Date start;
+
+void tprint(msg) {
+  print('###$msg');
+}
+
+class Macros {
+  static const String testTime = '<TIME>';
+  static const String testfile = '<FILENAME>';
+  static const String testGroup = '<GROUPNAME>';
+  static const String testDescription = '<TESTNAME>';
+  static const String testMessage = '<MESSAGE>';
+  static const String testStacktrace = '<STACK>';
+}
+
+String formatMessage(filename, groupname,
+    [ testname = '', testTime = '', result = '',

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

nit: no extra space after [ and before ]

[gram, 2012/09/20 18:58:03]

Done.

+      message = '', stack = '' ]) {
+  var format = errorFormat;
+  if (result == 'pass') format = passFormat;
+  else if (result == 'fail') format = failFormat;
+  return format.
+      replaceAll(Macros.testTime, testTime).
+      replaceAll(Macros.testfile, filename).
+      replaceAll(Macros.testGroup, groupname).
+      replaceAll(Macros.testDescription, testname).
+      replaceAll(Macros.testMessage, message).
+      replaceAll(Macros.testStacktrace, stack);
+}
+
+void outputResult(start, label, result, [message = '']) {
+  var idx = label.lastIndexOf('###');
+  var group = '', test = '';
+  if (idx >= 0) {
+    group = '${label.substring(0, idx).replaceAll("###", " ")} ';
+    test = '${label.substring(idx+3)} ';
+  } else {
+    test = '$label ';
+  }
+  var elapsed = '';
+  if (includeTime) {
+    var end = new Date.now();
+    double duration = (end.difference(start)).inMilliseconds.toDouble();
+    duration /= 1000;
+    elapsed = '${duration.toStringAsFixed(3)}s ';
+  }
+  tprint(formatMessage('$testfile ', group, test, elapsed, result, message));
+}
+
+pass(start, label) {
+  ++passCount;
+  outputResult(start, label, 'pass');
+}
+
+fail(start, label, message) {
+  ++failCount;
+  outputResult(start, label, 'fail', message);
+}
+
+error(start, label, message) {
+  ++errorCount;
+  outputResult(start, label, 'error', message);
+}
+
+void printSummary(String testFile, int passed, int failed, int errors,
+    [String uncaughtError = '']) {
+  tprint('');
+  if (passed == 0 && failed == 0 && errors == 0) {
+    tprint('$testFile: No tests found.');
+  } else if (failed == 0 && errors == 0 && uncaughtError == null) {
+    tprint('$testFile: All $passed tests passed.');
+  } else {
+    if (uncaughtError != null) {
+      tprint('$testFile: Top-level uncaught error: $uncaughtError');
+    }
+    tprint('$testFile: $passed PASSED, $failed FAILED, $errors ERRORS');
+  }
+}
+
+complete() {
+  if (summarize) {
+    printSummary(testfile, passCount, failCount, errorCount);
+  }
+  exit(failCount > -0 ? -1 : 0);

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

-0 ?

[gram, 2012/09/20 18:58:03]

Done.

+}
+
+runTextLayoutTest(testNum) {
+  var url = '$baseUrl?test=$testNum';
+  var stdout = new List();
+  start = new Date.now();
+  var process = Process.start(drt, [ url ]);

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

nit: [ url ] => [url] (no spaces)

[gram, 2012/09/20 18:58:03]

Done.

+  StringInputStream stdoutStringStream = new StringInputStream(process.stdout);
+  stdoutStringStream.onLine = () {
+    if (stdoutStringStream.closed) return;
+    var line = stdoutStringStream.readLine();
+    while (null != line) {
+      stdout.add(line);
+      line = stdoutStringStream.readLine();

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

I feel like this logic occurs a lot, is there any library code (or can we create
a common utility method) that takes a stream and returns a Future<List<String>>?

[gram, 2012/09/20 18:58:03]

It would be nice if the IO library had that; I don't think it belongs here
though.

[Siggi Cherem (dart-lang), 2012/09/20 19:56:27]

We could put it in utils.dart for now?

+    }
+  };
+  process.onExit = (exitCode) {
+    process.close();
+    if (stdout.length > 0 && stdout[stdout.length-1].startsWith('#EOF')) {
+      stdout.removeLast();
+    }
+    var done = false;
+    var i = 0;
+    var label = null;
+    var labelMarker = 'CONSOLE MESSAGE: #TEST ';
+    var contentMarker = 'layer at ';
+    while (i < stdout.length) {
+      if (label == null && stdout[i].startsWith(labelMarker)) {
+        label = stdout[i].substring(labelMarker.length);
+        if (label == 'NONEXISTENT') {
+          complete();
+        }
+      } else if (stdout[i].startsWith(contentMarker)) {
+        if (label == null) {
+          complete();
+        }
+        var expectedFileName =
+            '$sourceDir${Platform.pathSeparator}'
+            '${label.replaceAll("###", "_")
+                     .replaceAll(const RegExp("[^A-Za-z0-9]"),"_")}.txt';
+        var expected = new File(expectedFileName);
+        if (regenerate) {
+          var ostream = expected.openOutputStream(FileMode.WRITE);
+          while (i < stdout.length) {
+            ostream.writeString(stdout[i]);
+            ostream.writeString('\n');
+            i++;
+          }
+          ostream.close();
+          pass(start, label);
+        } else {
+          if (!expected.existsSync()) {

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

combine with prev line (flatten):
} else if (!expected.existsSync()) {
 ...
} else {

}

[gram, 2012/09/20 18:58:03]

Done.

+            fail(start, label, 'No expectation file');
+          } else {
+            var lines = expected.readAsLinesSync();
+            if (lines.length != stdout.length - i) {
+              fail(start, label, 'Expectation file has wrong length');

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

it's still useful to show some diffs in this case (e.g. some times the
difference is an empty line in the middle). 

[gram, 2012/09/20 18:58:03]

Done.

+            } else {
+              var match = true;
+              for (var j = 0; j < lines.length; j++) {
+                if (lines[j] != stdout[i+j]) {
+                  fail(start, label, 'Expectation differs at line ${j+1}');
+                  match = false;
+                  break;
+                }
+              }
+              if (match) pass(start, label);
+            }
+          }
+        }
+        done = true;
+        break;
+      }
+      i++;
+    }
+    if (label != null) {
+      if (!done) error(start, label, 'Failed to parse output');
+      runTextLayoutTest(testNum + 1);
+    }
+  };
+}
+
+runPixelLayoutTest(int testNum) {
+  var url = '$baseUrl?test=$testNum';
+  var stdout = new List();
+  start = new Date.now();
+  var process = Process.start(drt, [ "$url'-p" ]);

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

nit: [ "$url'-p" ] => ["$url'-p"] (no spaces)

[gram, 2012/09/20 18:58:03]

Done.

+  ListInputStream stdoutStream = process.stdout;
+  stdoutStream.onData = () {
+    if (!stdoutStream.closed) {
+      var data = stdoutStream.read();
+      stdout.addAll(data);
+    }
+  };
+  stdoutStream.onError = (e) {
+    print(e);
+  };
+  process.onExit = (exitCode) {
+    stdout.addAll(process.stdout.read());
+    process.close();
+    var labelMarker = 'CONSOLE MESSAGE: #TEST ';
+    var contentMarker = 'Content-Length: ';
+    var eol = '\n'.charCodeAt(0);
+    var pos = -1;
+    var label = null;
+    var done = false;
+
+    while(pos < stdout.length) {
+      var idx = stdout.indexOf(eol, ++pos);
+      if (idx < 0) break;
+      StringBuffer sb = new StringBuffer();
+      for (var i = pos; i < idx; i++) {
+        sb.addCharCode(stdout[i]);
+      }
+      var line = sb.toString();
+
+      if (label == null && line.startsWith(labelMarker)) {
+        label = line.substring(labelMarker.length);
+        if (label == 'NONEXISTENT') {
+          complete();
+        }
+      } else if (line.startsWith(contentMarker)) {
+        if (label == null) {
+          complete();
+        }
+        var len = int.parse(line.substring(contentMarker.length));
+        pos = idx+1;

[Siggi Cherem (dart-lang), 2012/09/20 17:37:27]

nit: spaces around + 

[gram, 2012/09/20 18:58:03]

Done.

+        var expectedFileName =
+            '$sourceDir${Platform.pathSeparator}'
+            '${label.replaceAll("###","_").
+                     replaceAll(const RegExp("[^A-Za-z0-9]"),"_")}.png';
+        var expected = new File(expectedFileName);
+        if (regenerate) {
+          var ostream = expected.openOutputStream(FileMode.WRITE);
+          ostream.writeFrom(stdout, pos, len);
+          ostream.close();
+          pass(start, label);
+        } else {
+          if (!expected.existsSync()) {
+            fail(start, label, 'No expectation file');
+          } else {
+            var bytes = expected.readAsBytesSync();
+            if (bytes.length != len) {
+              fail(start, label, 'Expectation file has wrong length');
+            } else {
+              var match = true;
+              for (var j = 0; j < len; j++) {
+                if (bytes[j] != stdout[pos+j]) {
+                  fail(start, label, 'Expectation differs at byte ${j+1}');
+                  match = false;
+                  break;
+                }
+              }
+              if (match) pass(start, label);
+            }
+          }
+        }
+        done = true;
+        break;
+      }
+      pos = idx;
+    }
+    if (label != null) {
+      if (!done) error(start, label, 'Failed to parse output');
+      runPixelLayoutTest(testNum + 1);
+    }
+  };
+}