Testrunner for 3rd parties.

utils/testrunner/dart_wrap_task.dart

+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// The DartWrapTask generates a Dart wrapper for a test file, that has a

[Siggi Cherem (dart-lang), 2012/08/29 20:47:35]

use /** */ so we make all these comments valid dartdocs.

[Siggi Cherem (dart-lang), 2012/08/30 16:46:40]

note that this and the comment below are still pending...

[gram, 2012/08/30 17:34:05]

Done.

+// test Configuration customized for the options specified by the user.
+class DartWrapTask extends PipelineTask {
+  String sourceFileTemplate;
+  String tempDartFileTemplate;
+
+  DartWrapTask(this.sourceFileTemplate, this.tempDartFileTemplate);
+
+  void execute(Path testfile, List stdout, List stderr, bool verboseLogging,
+              Function exitHandler) {
+    // Get the source test file and canonicalize the path.
+    var sourceName = makePathAbsolute(expandMacros(sourceFileTemplate, testfile));

[Siggi Cherem (dart-lang), 2012/08/29 20:47:35]

80 col

[gram, 2012/08/30 17:34:05]

Done.

+    // Get the destination file.
+    var destFile = expandMacros(tempDartFileTemplate, testfile);
+
+    // Working buffer for the Dart wrapper.
+    StringBuffer sbuf = new StringBuffer();
+
+    // Add the common header stuff.
+    var p = new Path(sourceName);
+    sbuf.add(directives(p.filenameWithoutExtension,
+                        config.unittestPath,
+                        sourceName));
+
+    // Add the test configuration and determine the action function.
+    var action;
+    if (config.listTests) {
+      action = 'listTests';
+      sbuf.add(barebonesConfig());
+      sbuf.add(listTestsFunction);
+      sbuf.add(formatListMessageFunction(config.listFormat));
+    } else if (config.listGroups) {
+      sbuf.add(barebonesConfig());
+      sbuf.add(listGroupsFunction);
+      sbuf.add(formatListMessageFunction(config.listFormat));
+      action = 'listGroups';
+    } else {
+      if (config.runInBrowser) {
+        sbuf.add(browserTestPrintFunction);
+      } else {
+        sbuf.add(nonBrowserTestPrintFunction);
+      }
+
+      if (config.includeTime) {
+        sbuf.add(elapsedFunction);
+      } else {
+        sbuf.add(dummyElapsedFunction);
+      }
+      sbuf.add(dumpTestResultFunction);
+      sbuf.add(testConfig(sourceName,
+                          config.runInBrowser,
+                          config.runIsolated,
+                          config.immediateOutput,
+                          config.produceSummary));
+      // Add the isolate stuff, if applicable.
+      if (config.runIsolated) {
+        sbuf.add(runIsolateTestsFunction);
+        action = 'runIsolateTests';
+      } else {
+        sbuf.add(runTestsFunction);
+        action = 'runTests';
+      }
+      sbuf.add(formatMessageFunction(config.passFormat,
+                                     config.failFormat,
+                                     config.errorFormat));
+    }
+
+    // Add the filter, if applicable.
+    if (config.filtering) {
+      if (config.includeFilter.length > 0) {
+        sbuf.add(filterTestFunction(config.includeFilter, 'true'));
+      } else {
+        sbuf.add(filterTestFunction(config.excludeFilter, 'false'));
+      }
+    }
+
+    // Add the common trailer stuff.
+    sbuf.add(dartMain(sourceName, action, config.filtering));
+
+    // Save the Dart file.
+    createFile(destFile, sbuf.toString());
+    exitHandler(0);
+  }
+
+  String directives(String library, String unittest, String sourceName) {
+    return """
+#library('$library');
+#import('dart:isolate');
+#import('$unittest', prefix:'unittest');
+#import('$sourceName', prefix: 'test');
+""";
+  }
+
+  // For 'printing' when we are in the browser, we add text elements
+  // to a DOM element with id 'console'.
+  final String browserTestPrintFunction = """
+#import('dart:html');
+void tprint(msg) {
+var pre = query('#console');
+pre.elements.add(new Text('###\$msg\\n'));
+}
+""";
+
+  // For printing when not in the browser we can just use Dart's print().
+  final String nonBrowserTestPrintFunction = """
+void tprint(msg) {
+print('###\$msg');
+}
+""";
+
+  // The core skeleton for a config. Most of the guts is in the
+  // parameter [body].
+  String configuration(String body) {
+    return """
+class TestRunnerConfiguration extends unittest.Configuration {
+  get name => 'Test runner configuration';
+  get autoStart => false;
+  $body
+}
+""";
+  }
+
+  // A function to give us the elapsed time for a test.
+  final String elapsedFunction = """
+String elapsed(TestCase t) {
+  double duration = t.runningTime.inMilliseconds;
+  duration /= 1000;
+  return '\${duration.toStringAsFixed(3)}s ';
+}
+""";
+
+  // A dummy version of the elapsed function for when the user
+  // doesn't want test times included.
+  final String dummyElapsedFunction = """
+String elapsed(TestCase t) {
+  return '';
+}
+""";
+
+  // A function to print the results of a test.
+  final String dumpTestResultFunction = """
+void dumpTestResult(source, TestCase t) {
+  var groupName = '', testName = '';
+  var idx = t.description.lastIndexOf('###');
+  if (idx >= 0) {
+      groupName = t.description.substring(0, idx).replaceAll('###', ' ');
+      testName = t.description.substring(idx+3);
+  } else {
+      testName = t.description;
+  }
+  var stack = (t.stackTrace == null) ? '' : '\${t.stackTrace} ';
+  var message = (t.message.length > 0) ? '\$t.message ' : '';
+  var duration = elapsed(t);
+  tprint(formatMessage(source, '\$groupName ', '\$testName ', 
+      duration, t.result, message, stack));
+}
+""";
+
+  // A barebones config, used for listing tests, not running them.
+  String barebonesConfig() {
+    return configuration('');
+  }
+
+  // A much more complex config, used for running tests.
+  String testConfig(String sourceName,
+                    bool runInBrowser,
+                    bool runIsolated,
+                    bool immediateOutput,
+                    bool summary) {
+    StringBuffer sbuf = new StringBuffer();
+
+    if (runIsolated) {
+      // Add a constructor that saves the isolate port.
+      sbuf.add("  var port;\n  TestRunnerConfiguration(this.port);\n");
+    }
+
+    if (immediateOutput) {
+      // Output test results in onTestResult instead of onDone.
+      sbuf.add("""
+  void onTestResult(TestCase testCase) {
+    dumpTestResult('${sourceName} ', testCase);
+  }
+
+  void onDone(int passed, int failed, int errors, List<TestCase> results,
+      String uncaughtError) {
+""");
+    } else {
+      // Not immediate; we output test results in onDone.
+      sbuf.add("""
+  void onDone(int passed, int failed, int errors, List<TestCase> results,
+      String uncaughtError) {
+    // Print each result.
+    for (final testCase in results) {
+      dumpTestResult('${sourceName} ', testCase);
+    }
+""");
+    }
+
+    if (summary) {
+      // Add the code to produce the summary for the test file.
+      sbuf.add("""
+    tprint('');
+    var success = false;
+    if (passed == 0 && failed == 0 && errors == 0) {
+      tprint('$sourceName: No tests found.');
+    } else if (failed == 0 && errors == 0 && uncaughtError == null) {
+      tprint('$sourceName: All \$passed tests passed.');
+      success = true;
+    } else {
+      if (uncaughtError != null) {
+        tprint('$sourceName: Top-level uncaught error: \$uncaughtError');
+      }
+      tprint('$sourceName: \$passed PASSED, \$failed FAILED, \$errors ERRORS');
+    }
+""");
+    }
+
+    if (runIsolated) {
+      // Add the code to tell the master whether we succeeded or not.
+      sbuf.add("""
+    var success = (passed > 0 && failed == 0 && errors == 0 && 
+        uncaughtError == null);
+    port.send(success);
+""");
+    } else if (runInBrowser) {
+      // Add the code to tell DRT we are done and it can exit.
+      sbuf.add("    window.postMessage('done', '*');\n");
+    }
+    sbuf.add("  }\n");
+    return configuration(sbuf.toString());
+  }
+
+  // A simple format function for listing tests.
+  String formatListMessageFunction(String format) {
+    return """
+String formatMessage(filename, groupname, [ testname = '']) {
+  return '${format}'.
+      replaceAll('${Macros.testfile}', filename).
+      replaceAll('${Macros.testGroup}', groupname).
+      replaceAll('${Macros.testDescription}', testname);
+}
+""";
+  }
+
+  // A richer format function for test results.
+  String formatMessageFunction(
+      String passFormat, String failFormat, String errorFormat) {
+    return """
+String formatMessage(filename, groupname,
+    [ testname = '', testTime = '', result = '',
+      message = '', stack = '' ]) {
+  var format = '$errorFormat';
+  if (result == 'pass') format = '$passFormat';
+  else if (result == 'fail') format = '$failFormat';
+  return format.
+      replaceAll('${Macros.testTime}', testTime).
+      replaceAll('${Macros.testfile}', filename).
+      replaceAll('${Macros.testGroup}', groupname).
+      replaceAll('${Macros.testDescription}', testname).
+      replaceAll('${Macros.testMessage}', message).
+      replaceAll('${Macros.testStacktrace}', stack);
+}
+""";
+  }
+
+  // A function to list the test groups.
+  final String listGroupsFunction = """
+listGroups(testfile) {
+  List tests = unittest.testCases;
+  Map groups = {};
+  for (var t in tests) {
+    var groupName, testName = '';
+    var idx = t.description.lastIndexOf('###');
+    if (idx >= 0) {
+      groupName = t.description.substring(0, idx).replaceAll('###', ' ');
+      if (!groups.containsKey(groupName)) {
+        groups[groupName] = '';
+      }
+    }
+  }
+  for (var g in groups.getKeys()) {
+    var msg = formatMessage('\$testfile ', '\$g ');
+    print('###\$msg');
+  }
+}
+""";
+
+  // A function to list the tests.
+  final String listTestsFunction = """
+    listTests(testfile) {
+      List tests = unittest.testCases;
+      for (var t in tests) {
+        var groupName, testName = '';
+        var idx = t.description.lastIndexOf('###');
+        if (idx >= 0) {
+          groupName = t.description.substring(0, idx).replaceAll('###', ' ');
+          testName = t.description.substring(idx+3);
+        } else {
+          groupName = '';
+          testName = t.description;
+        }
+        var msg = formatMessage('\$testfile ', '\$groupName ',
+            '\$testName ');
+        print('###\$msg');
+      }
+    }
+""";
+
+  // A function to filter the tests.
+  String filterTestFunction(List filters, String filterReturnValue) {
+    StringBuffer sbuf = new StringBuffer();
+    sbuf.add('filterTest(t) {\n');
+    if (filters != null) {
+      sbuf.add('  var name = t.description.replaceAll("###", " ");\n');
+      for (var f in filters) {
+        sbuf.add('  if (name.indexOf("$f")>=0) return $filterReturnValue;\n');
+      }
+      sbuf.add('  return !$filterReturnValue;\n');
+    } else {
+      // TODO - is this right? Should it be filterReturnValue?
+      sbuf.add('  return true;\n');
+    }
+    sbuf.add('}\n');
+    return sbuf.toString();
+  }
+
+  // Code to support running single tests as master/slave in isolates.
+  final String runIsolateTestsFunction = """
+runSlaveTest() {
+  port.receive((testName, sendport) {
+    unittest.configure(new TestRunnerConfiguration(sendport));    
+    unittest.group('', test.main);
+    unittest.filterTests(testName);
+    unittest.runTests();    
+  });
+}
+
+var testNum;
+var failed;
+
+runMasterTest() {
+  var tests = unittest.testCases;
+  SendPort sport = spawnFunction(runSlaveTest);
+  sport.call(tests[testNum].description).then((result) {
+    if (!result) failed = true;
+    ++testNum;
+    if (testNum < tests.length) {
+      runMasterTest();
+    } else if (failed) {
+      throw new Exception("Some tests failed.");      
+    }
+  });
+}
+
+runIsolateTests(f) {
+  testNum = 0;
+  failed = false;
+  runMasterTest();
+}
+""";
+
+  // Code for running all tests in the normal (non-isolate) way.
+  final String runTestsFunction = "runTests(f) { unittest.runTests(); }\n";
+
+  // The main function, that creates the config, filters the tests if
+  // necessary, then performs the action (list/run/run-isolated).
+  String dartMain(String sourceName, String action, bool filter) {
+    return """
+main() {
+  unittest.groupSep = '###';
+  unittest.configure(new TestRunnerConfiguration());
+  unittest.group('', test.main);
+  ${filter?'unittest.filterTests(filterTest);':''}
+  $action('$sourceName');
+}
+""";
+  }
+
+}