Support for pixel layout tests.

utils/testrunner/dart_wrap_task.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * The DartWrapTask generates a Dart wrapper for a test file, that has a
  * test Configuration customized for the options specified by the user.
  */
 class DartWrapTask extends PipelineTask {
   final String _sourceFileTemplate;
   final String _tempDartFileTemplate;
 
   DartWrapTask(this._sourceFileTemplate, this._tempDartFileTemplate);
 
   void execute(Path testfile, List stdout, List stderr, bool logging,
               Function exitHandler) {
     // Get the source test file and canonicalize the path.
     var sourceName = makePathAbsolute(
         expandMacros(_sourceFileTemplate, testfile));
     // Get the destination file.
     var destFile = expandMacros(_tempDartFileTemplate, testfile);
 
     var p = new Path(sourceName);
-    if (isLayoutRenderTest(sourceName) || config.generateRenders) {
-      makeLayoutTestWrapper(sourceName, destFile, p.filenameWithoutExtension);
-      exitHandler(0);
-      return;
+
+    if (config.layoutText || config.layoutPixel) {
+      // Layout tests are more complicated, as we have a 'meta'
+      // wrapped Dart program that in turn invokes DRT to run the

[Siggi Cherem (dart-lang), 2012/09/14 20:01:23]

we've talked about this before, but I think we should revisit the question of
whether we want to have this feature of running test suites as layout tests - it
adds a lot of complexity in the test runner (compared to just treating a layout
test as a stand-alone program) and I'm not so sure how much people will find it
useful.

Regardless of what we decide, I also think we should support the other scenario,
that is, letting people write a layout test without using unittest all together
(run this program in the browser, return what it looks like).

+      // normal wrapped Dart program, once for each test. So we generate
+      // a layout test wrapper file for the real tests and a controller
+      // wrapper to manage overall execution.
+      var childFile = '${destFile.substring(0, destFile.length-5)}-child.dart';

[Siggi Cherem (dart-lang), 2012/09/14 20:01:23]

style: please add spaces around operators (destFile.length - 5).

[gram, 2012/09/19 23:24:38]

Done.

+      makeLayoutTestWrapper(sourceName, childFile,
+          p.filenameWithoutExtension);
+      makeLayoutTestControllerWrapper(sourceName, destFile, childFile,
+          p.filenameWithoutExtension,
+          sourceName.substring(0, sourceName.length-5)); // Strip .dart.

[Siggi Cherem (dart-lang), 2012/09/14 20:01:23]

ditto

[gram, 2012/09/19 23:24:38]

Done.

+    } else {
+      makeNonLayoutTestWrapper(sourceName, destFile,
+                               p.filenameWithoutExtension);
     }
+    exitHandler(0);
+  }
+
+  void makeNonLayoutTestWrapper(String sourceName, String destFile,
+                           String sourceNameWithoutExtension) {
 
     // Working buffer for the Dart wrapper.
     StringBuffer sbuf = new StringBuffer();
 
     // Add the common header stuff.
-    sbuf.add(directives(p.filenameWithoutExtension,
+    sbuf.add(directives(sourceNameWithoutExtension,
                         config.unittestPath,
                         sourceName));
 
     // Add the test configuration and determine the action function.
     var action;
     if (config.listTests) {
       action = 'listTests';
       sbuf.add(barebonesConfig());
       sbuf.add(listTestsFunction);
       sbuf.add(formatListMessageFunction(config.listFormat));
@@ skipping 46 matching lines @@
       } else {
         sbuf.add(filterTestFunction(config.excludeFilter, 'false'));
       }
     }
 
     // Add the common trailer stuff.
     sbuf.add(dartMain(sourceName, action, config.filtering));
 
     // Save the Dart file.
     createFile(destFile, sbuf.toString());
-    exitHandler(0);
   }
 
   void cleanup(Path testfile, List stdout, List stderr,
                bool logging, bool keepFiles) {
     deleteFiles([_tempDartFileTemplate], testfile, logging, keepFiles, stdout);
   }
 
   void makeLayoutTestWrapper(String sourceName, String destFile,
                              String libraryName) {
     StringBuffer sbuf = new StringBuffer();
-    var cfg = config.unittestPath.
-        replaceAll('unittest.dart', 'html_layout_config.dart');
     sbuf.add("""
 #library('$libraryName');
 #import('dart:math');
 #import('dart:isolate');
 #import('dart:html');
+#import('dart:uri');
 #import('${config.unittestPath}', prefix:'unittest');
-#import('$cfg', prefix:'unittest');
 #import('$sourceName', prefix: 'test');
 """);
+
     // Add the filter, if applicable.
     if (config.filtering) {
       if (config.includeFilter.length > 0) {
         sbuf.add(filterTestFunction(config.includeFilter, 'true'));
       } else {
         sbuf.add(filterTestFunction(config.excludeFilter, 'false'));
       }
     }
     sbuf.add("""
+
+class LayoutTestConfiguration extends unittest.Configuration {
+  get autoStart => false;
+  void onTestResult(TestCase testCase) {
+      window.postMessage('done', '*'); // Unblock DRT
+  }
+}
+
 main() {
-unittest.groupSep = '###';
-unittest.useHtmlLayoutConfiguration();
-unittest.group('', test.main);
-${config.filtering ? 'unittest.filterTests(filterTest);' : ''}
-if (window.location.search == '') unittest.runTests();
+  unittest.groupSep = '###';
+  unittest.configure(new LayoutTestConfiguration());
+  // Create the set of test cases.
+  unittest.group('', test.main);
+  // Do any user-specified test filtering.
+  ${config.filtering ? 'unittest.filterTests(filterTest);' : ''}
+  // Filter to the test number in the search query.
+  var testNum = parseInt(window.location.search.substring(6));
+  if (testNum < 0 || testNum >= unittest.testCases.length) {
+    print('#TEST NONEXISTENT');
+  } else {
+    var name = unittest.testCases[testNum].description;
+    print('#TEST \$name');
+    unittest.filterTests(name);
+    // Run the test.
+    unittest.runTests();
+  }
 }  
 """);
     // Save the Dart file.
     createFile(destFile, sbuf.toString());
   }
 
+  void makeLayoutTestControllerWrapper(String sourceName,
+      String destName, String childName, String libraryName,
+      String expectedDirectory) {
+    if (config.regenerate) {
+      var d = new Directory(expectedDirectory);
+      if (!d.existsSync()) {
+        d.createSync();
+      }
+    }
+    StringBuffer sbuf = new StringBuffer();
+    var htmlFile = childName.replaceFirst('-child.dart', '.html');
+    // Add common prefix.
+    sbuf.add("""

[Siggi Cherem (dart-lang), 2012/09/14 20:01:23]

could we move a lot of this 'generated code' into a runtime library?

Basically, generate little code and call into a library of the testrunner
runtime?

+#library('$libraryName');
+#import('dart:uri');
+#import('dart:io');
+#import('dart:math');
+
+var label;
+
+main() {
+  runTest(0);
+}
+
+var passCount = 0, failCount = 0, errorCount = 0;
+
+outputResult(start, label, result, [message = '']) {
+  var idx = label.lastIndexOf('###');
+  var group = '', test = '';
+  if (idx >= 0) {
+    group = '\${label.substring(0, idx).replaceAll("###", " ")} ';
+    test = '\${label.substring(idx+3)} ';
+  } else {
+    test = '\$label ';
+  }
+  var elapsed = '';
+  if (${config.includeTime}) {
+    var end = new Date.now(); 
+    double duration = (end - start).inMilliseconds.toDouble();
+    duration /= 1000;
+    elapsed = '\${duration.toStringAsFixed(3)}s ';
+  }
+  tprint(formatMessage('$sourceName ', group, test, elapsed, result, message));
+}
+
+pass(start, label) {
+  ++passCount;
+  outputResult(start, label, 'pass');
+}
+
+fail(start, label, message) {
+  ++failCount;
+  outputResult(start, label, 'fail', message);
+}
+
+error(start, label, message) {
+  ++errorCount;
+  outputResult(start, label, 'error', message);
+}
+
+complete() {
+  printSummary('$sourceName', passCount, failCount, errorCount); 
+  exit(failCount > -0 ? -1 : 0);
+}
+
+""");
+
+    sbuf.add(nonBrowserTestPrintFunction);
+    sbuf.add(formatMessageFunction(config.passFormat,
+                                   config.failFormat,
+                                   config.errorFormat));
+
+    sbuf.add(config.produceSummary ?
+        printSummaryFunction : stubPrintSummaryFunction);
+
+    if (config.layoutText) {
+      sbuf.add("""
+runTest(testNum) {
+  var url = 'file://$htmlFile?test=\$testNum';
+  var stdout = new List();
+  Date start = new Date.now();
+  var process = Process.start('${config.drtPath}', [ url ]);
+  StringInputStream stdoutStringStream = new StringInputStream(process.stdout);
+  stdoutStringStream.onLine = () {
+    if (stdoutStringStream.closed) return;
+    var line = stdoutStringStream.readLine();
+    while (null != line) {
+      stdout.add(line);
+      line = stdoutStringStream.readLine();
+    }
+  };
+  process.onExit = (exitCode) {
+    process.close();
+    if (stdout.length > 0 && stdout[stdout.length-1].startsWith('#EOF')) {
+      stdout.removeLast();
+    }
+    var done = false;
+    var i = 0;
+    var label = null;
+    var labelMarker = 'CONSOLE MESSAGE: #TEST ';
+    var contentMarker = 'layer at ';   
+    while (i < stdout.length) {
+      if (label == null && stdout[i].startsWith(labelMarker)) {
+        label = stdout[i].substring(labelMarker.length);
+        if (label == 'NONEXISTENT') {
+          complete();
+        }
+      } else if (stdout[i].startsWith(contentMarker)) {
+        if (label == null) {
+          complete();
+        }
+        var expectedFileName = 
+            '$expectedDirectory${Platform.pathSeparator}'
+            '\${label.replaceAll("###", "_")
+                     .replaceAll(const RegExp("[^A-Za-z0-9]"),"_")}.txt';
+        var expected = new File(expectedFileName);   
+        if (${config.regenerate}) {
+          var ostream = expected.openOutputStream(FileMode.WRITE);
+          while (i < stdout.length) {
+            ostream.writeString(stdout[i]);
+            ostream.writeString('\\n');
+            i++;
+          }
+          ostream.close();
+          pass(start, label);
+        } else {
+          if (!expected.existsSync()) {
+            fail(start, label, 'No expectation file');
+          } else {
+            var lines = expected.readAsLinesSync();
+            if (lines.length != stdout.length - i) {
+              fail(start, label, 'Expectation file has wrong length');
+            } else {
+              var match = true;
+              for (var j = 0; j < lines.length; j++) {
+                if (lines[j] != stdout[i+j]) {
+                  fail(start, label, 'Expectation differs at line \${j+1}');
+                  match = false;
+                  break;
+                }
+              }
+              if (match) pass(start, label);
+            }
+          }
+        } 
+        done = true;
+        break;
+      }
+      i++;
+    }
+    if (label != null) {
+      if (!done) error(start, label, 'Failed to parse output');
+      runTest(testNum+1);
+    }
+  };
+}
+""");
+    } else {
+      sbuf.add("""
+runTest(testNum) {
+  var url = 'file://$htmlFile?test=\$testNum';
+  var stdout = new List();
+  Date start = new Date.now();
+  var process = Process.start('${config.drtPath}', [ "\$url'-p" ]);
+  ListInputStream stdoutStream = process.stdout;
+  stdoutStream.onData = () {
+    if (!stdoutStream.closed) {
+      var data = stdoutStream.read();
+      stdout.addAll(data);
+    }
+  };
+  stdoutStream.onError = (e) {
+    print(e);
+  };
+  process.onExit = (exitCode) {
+      stdout.addAll(process.stdout.read());
+    process.close();
+    var labelMarker = 'CONSOLE MESSAGE: #TEST ';
+    var contentMarker = 'Content-Length: ';
+    var eol = '\\n'.charCodeAt(0);
+    var pos = -1;
+    var label = null;
+    var done = false; 
+
+    while(pos < stdout.length) {
+      var idx = stdout.indexOf(eol, ++pos);
+      if (idx < 0) break;
+      StringBuffer sb = new StringBuffer();
+      for (var i = pos; i < idx; i++) { 
+        sb.addCharCode(stdout[i]);
+      }
+      var line = sb.toString();
+       
+      if (label == null && line.startsWith(labelMarker)) {
+        label = line.substring(labelMarker.length);
+        if (label == 'NONEXISTENT') {
+          complete();
+        }
+      } else if (line.startsWith(contentMarker)) {
+        if (label == null) {
+          complete();
+        }
+        var len = parseInt(line.substring(contentMarker.length));
+        pos = idx+1;
+        var expectedFileName = 
+            '$expectedDirectory${Platform.pathSeparator}'
+            '\${label.replaceAll("###","_").
+                      replaceAll(const RegExp("[^A-Za-z0-9]"),"_")}.png';
+        var expected = new File(expectedFileName);   
+        if (${config.regenerate}) {
+          var ostream = expected.openOutputStream(FileMode.WRITE);
+          ostream.writeFrom(stdout, pos, len);
+          ostream.close();
+          pass(start, label);
+        } else {
+          if (!expected.existsSync()) {
+            fail(start, label, 'No expectation file');
+          } else {
+            var bytes = expected.readAsBytesSync();
+            if (bytes.length != len) {
+              fail(start, label, 'Expectation file has wrong length');
+            } else {
+              var match = true;
+              for (var j = 0; j < len; j++) {
+                if (bytes[j] != stdout[pos+j]) {
+                  fail(start, label, 'Expectation differs at byte \${j+1}');
+                  match = false;
+                  break;
+                }
+              }
+              if (match) pass(start, label);
+            }
+          }
+        } 
+        done = true;
+        break;
+      }
+      pos = idx;
+    }
+    if (label != null) {
+      if (!done) error(start, label, 'Failed to parse output');
+      runTest(testNum+1);
+    }
+  };
+}
+""");
+    }
+    createFile(destName, sbuf.toString());
+  }
+
   String directives(String library, String unittest, String sourceName) {
     return """
 #library('$library');
 #import('dart:math');
 #import('dart:isolate');
 #import('$unittest', prefix:'unittest');
 #import('$sourceName', prefix: 'test');
 """;
   }
 
@@ skipping 97 matching lines @@
   var message = (t.message.length > 0) ? '\$t.message ' : '';
   var duration = elapsed(t);
   tprint(formatMessage(source, '\$groupName ', '\$testName ', 
       duration, t.result, message, stack));
 }
 """;
 
   // A function to print the test summary.
   final String printSummaryFunction = """
 void printSummary(String testFile, int passed, int failed, int errors,
-    String uncaughtError) { 
+    [String uncaughtError = '']) { 
   tprint('');
   if (passed == 0 && failed == 0 && errors == 0) {
     tprint('\$testFile: No tests found.');
   } else if (failed == 0 && errors == 0 && uncaughtError == null) {
     tprint('\$testFile: All \$passed tests passed.');
   } else {
     if (uncaughtError != null) {
       tprint('\$testFile: Top-level uncaught error: \$uncaughtError');
     }
     tprint('\$testFile: \$passed PASSED, \$failed FAILED, \$errors ERRORS');
   }
 }
 """;
 
   final String stubPrintSummaryFunction = """
 void printSummary(String testFile, int passed, int failed, int errors,
-    String uncaughtError) {   
+    [String uncaughtError = '']) {   
 }
 """;
 
   // A function to print all test results.
   final String printAllTestResultsFunction = """
 void printResults(testfile, List<unittest.TestCase> results) {
   for (final testCase in results) {
     dumpTestResult('\$testfile ', testCase);
   }
 }
@@ skipping 188 matching lines @@
 """;
 
   // Code for running all tests in the normal (non-isolate) way.
   final String runTestsFunction = """
 runTests() {
   unittest.runTests();
 }
 """;
 
 }