Adds support for layout tests.

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
@@ skipping 501 matching lines @@
    * step and an execution step, both with the appropriate executable and
    * arguments.
    */
   void enqueueBrowserTest(TestInformation info,
                           String testName,
                           Set<String> expectations) {
     Map optionsFromFile = info.optionsFromFile;
     String filename = info.filename;
     if (optionsFromFile['isMultitest']) return;
     bool isWebTest = optionsFromFile['containsDomImport'];
+    List<String> webTestOutput = optionsFromFile['webTestOutput'];
     bool isLibraryDefinition = optionsFromFile['isLibraryDefinition'];
     if (!isLibraryDefinition && optionsFromFile['containsSourceOrImport']) {
       print('Warning for $filename: Browser tests require #library '
             'in any file that uses #import, #source, or #resource');
     }
 
     final String compiler = configuration['compiler'];
     final String runtime = configuration['runtime'];
     final String testPath =
         new File(filename).fullPathSync().replaceAll('\\', '/');
@@ skipping 17 matching lines @@
       if (!isWebTest) {
         // test.dart will import the dart test directly, if it is a library,
         // or indirectly through test_as_library.dart, if it is not.
         String dartLibraryFilename;
         if (isLibraryDefinition) {
           dartLibraryFilename = testPath;
         } else {
           dartLibraryFilename = 'test_as_library.dart';
           File file = new File('$tempDir/$dartLibraryFilename');
           RandomAccessFile dartLibrary = file.openSync(FileMode.WRITE);
-          dartLibrary.writeStringSync(WrapDartTestInLibrary(testPath));
+          dartLibrary.writeStringSync(wrapDartTestInLibrary(testPath));
           dartLibrary.closeSync();
         }
 
         File file = new File(dartWrapperFilename);
         RandomAccessFile dartWrapper = file.openSync(FileMode.WRITE);
         dartWrapper.writeStringSync(
-            DartTestWrapper(dartDir, dartLibraryFilename));
+            dartTestWrapper(dartDir, dartLibraryFilename));
         dartWrapper.closeSync();
       } else {
         dartWrapperFilename = testPath;
         // TODO(whesse): Once test.py is retired, adjust the relative path in
         // the client/samples/dartcombat test to its css file, remove the
         // "../../" from this path, and move this out of the isWebTest guard.
         // Also remove getHtmlName, and just use test.html.
         // TODO(efortuna): this shortening of htmlFilename is a band-aid until
         // the above TODO gets fixed. Windows cannot have paths that are longer
         // than 260 characters, and without this hack, we were running past the
         // the limit.
         String htmlFilename = getHtmlName(filename);
         while ('$tempDir/../$htmlFilename'.length >= 260) {
           htmlFilename = htmlFilename.substring(htmlFilename.length~/2);
         }
         htmlPath = '$tempDir/../$htmlFilename';
       }
       final String scriptPath = (compiler == 'none') ?
           dartWrapperFilename : compiledDartWrapperFilename;
       // Create the HTML file for the test.
       RandomAccessFile htmlTest = new File(htmlPath).openSync(FileMode.WRITE);
       String filePrefix = '';
       if (Platform.operatingSystem == 'windows') {
         // Firefox on Windows does not like absolute file path names that start
         // with 'C:' adding 'file:///' solves the problem.
         filePrefix = 'file:///';
       }
-      htmlTest.writeStringSync(GetHtmlContents(
+      String content;
+      if (webTestOutput.isEmpty()) {
+        content = getHtmlContents(
           filename,
           '$filePrefix$dartDir/lib/unittest/test_controller.js',
           scriptType,
-          '$filePrefix$scriptPath'));
+          '$filePrefix$scriptPath');
+      } else {
+        content = getHtmlLayoutContents(scriptType, '$filePrefix$scriptPath');
+      }
+      htmlTest.writeStringSync(content);
       htmlTest.closeSync();
 
       // Construct the command(s) that compile all the inputs needed by the
       // browser test. For running Dart in DRT, this will be noop commands.
       List<Command> commands = [];
       if (compiler != 'none') {
         commands.add(_compileCommand(
             dartWrapperFilename, compiledDartWrapperFilename,
             compiler, tempDir, vmOptions));
 
         // some tests require compiling multiple input scripts.
         List<String> otherScripts = optionsFromFile['otherScripts'];
         for (String name in otherScripts) {
-          int end = filename.lastIndexOf('/');
-          if (end == -1) {
+          String dir = _dirNameFromFile(filename);
+          if (dir == null) {
             print('Warning: error processing "OtherScripts" of $filename.');
             print('Skipping test ($testName).');
             return;
           }
-          String dir = filename.substring(0, end);
-          end = name.lastIndexOf('.dart');
+          int end = name.lastIndexOf('.dart');
           if (end == -1) {
             print('Warning: error processing "OtherScripts" in $filename.');
             print('Skipping test ($testName).');
             return;
           }
           String compiledName = '${name.substring(0, end)}.js';
           commands.add(_compileCommand(
               '$dir/$name', '$tempDir/$compiledName',
               compiler, tempDir, vmOptions));
         }
@@ skipping 19 matching lines @@
         if (runtime == 'drt' && compiler == 'none') {
           var dartFlags = ['--ignore-unrecognized-flags'];
           if (configuration["checked"]) {
             dartFlags.add('--enable_asserts');
             dartFlags.add("--enable_type_checks");
           }
           dartFlags.addAll(vmOptions);
           args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
         }
         args.add(htmlPath);
+        if (!webTestOutput.isEmpty()) {
+          String dir = _dirNameFromFile(filename);
+          args.add('--out-expectation=$dir/${webTestOutput[0]}');
+        }
       }
       commands.add(new Command('python', args));
 
       // Create BrowserTestCase and queue it.
       var testCase = new BrowserTestCase('$suiteName/$testName',
           commands, configuration, completeHandler, expectations,
           optionsFromFile['isNegative']);
       doTest(testCase);
     }
   }
 
+  String _dirNameFromFile(String filename) {
+    int end = filename.lastIndexOf('/');
+    return (end == -1) ? null : filename.substring(0, end);
+  }
+
   /** Helper to create a compilation command for a single input file. */
   Command _compileCommand(String inputFile, String outputFile,
       String compiler, String dir, var vmOptions) {
     String executable = TestUtils.compilerPath(configuration);
     List<String> args = TestUtils.standardOptions(configuration);
     switch (compiler) {
       case 'frog':
         String libdir = configuration['froglib'];
         if (libdir == '') {
           libdir = '$dartDir/frog/lib';
@@ skipping 165 matching lines @@
     Expect.isFalse(vmOptionsList.isEmpty(), "empty vmOptionsList");
     for (var vmOptions in vmOptionsList) {
       var options = new List<String>.from(vmOptions);
       options.addAll(args);
       result.add(options);
     }
 
     return result;
   }
 
+  /**
+   * Special options for individual tests are specified with comments directly
+   * in test files. Here is a list of parameters available:
+   *   - Flags can be passed to the vm or dartium process that runs the test as
+   *   follows:
+   *
+   *     // VMOptions=--flag1 --flag2
+   *
+   *   - Flags can be passed to the dart script that contains the test as
+   *   follows:
+   *
+   *     // DartOptions=--flag1 --flag2
+   *
+   *   - For tests that depend on compiling other files with dart2js (e.g.
+   *   isolate tests that use multiple source scripts), you can specify
+   *   additional files to compile as follows:
+   *
+   *     // OtherScripts=file1.dart file2.dart
+   *
+   *   - You can indicate whether a test is treated as a web-only test as
+   *   follows:
+   *
+   *     // WebTest=true

[Siggi Cherem (dart-lang), 2012/06/27 01:50:42]

this is something I'm adding on a separate CL
(https://chromiumcodereview.appspot.com//10685009), but I went ahead and started
writing the documentation right away assuming I'll submit this on a second step.

+   *
+   *   Most tests are not web tests, but can (and will be) wrapped within
+   *   another script file to test them also on browser environments (e.g.
+   *   language and corelib tests are run this way). Specifing this flag means
+   *   that no wrapping is necessary.
+   *
+   *   - Using test expectations for layout tests:
+   *
+   *     // WebTestOutput=expected_file.txt
+   *          or
+   *     // WebTestOutput=expected_file.png
+   *
+   *   By default tests are run without expectation files: test.dart assumes

[Bill Hesse, 2012/06/27 09:24:37]

Say "web tests" here, not just "tests".  This only applies to browser tests,
right?

[Siggi Cherem (dart-lang), 2012/06/27 20:21:05]

Done.

+   *   tests fail if the process return a non-zero exit code (in the case of web
+   *   tests, we check for PASS/FAIL indications in the test output). Specifying
+   *   a test output explicitly will check that the test output matches that
+   *   specified in the expectation file. In particular, the output of a web
+   *   test will be a rendering representation (either in text form if the
+   *   expectation ends in .txt, or an image, if the expectation ends in .png).
+   *   These expectations can be recorded for the first time by specifying the
+   *   option above, running the test, and running the copy command printed by
+   *   the test script.
+   */
   Map readOptionsFromFile(String filename) {
     RegExp testOptionsRegExp = const RegExp(@"// VMOptions=(.*)");
     RegExp dartOptionsRegExp = const RegExp(@"// DartOptions=(.*)");
     RegExp otherScriptsRegExp = const RegExp(@"// OtherScripts=(.*)");
+    RegExp testOutputRegExp = const RegExp(@"// WebTestOutput=(.*)");
     RegExp multiTestRegExp = const RegExp(@"/// [0-9][0-9]:(.*)");
     RegExp staticTypeRegExp =
         const RegExp(@"/// ([0-9][0-9]:){0,1}\s*static type warning");
     RegExp compileTimeRegExp =
         const RegExp(@"/// ([0-9][0-9]:){0,1}\s*compile-time error");
     RegExp staticCleanRegExp = const RegExp(@"// @static-clean");
     RegExp leadingHashRegExp = const RegExp(@"^#", multiLine: true);
     RegExp isolateStubsRegExp = const RegExp(@"// IsolateStubs=(.*)");
     RegExp domImportRegExp =
         const RegExp(@"^#import.*(dart:(dom|html)|html\.dart).*\)",
@@ skipping 39 matching lines @@
 
     matches = staticCleanRegExp.allMatches(contents);
     for (var match in matches) {
       if (isStaticClean) {
         throw new Exception(
             'More than one "// @static-clean=" line in test $filename');
       }
       isStaticClean = true;
     }
 
-    List<String> otherScripts = new List<String>();
-    matches = otherScriptsRegExp.allMatches(contents);
-    for (var match in matches) {
-      otherScripts.addAll(match[1].split(' ').filter((e) => e != ''));
+    List<String> otherScripts = _readListOption(otherScriptsRegExp, contents);
+    List<String> webTestOutput = _readListOption(testOutputRegExp, contents);
+    if (!webTestOutput.isEmpty() && webTestOutput.length > 1) {
+        throw new Exception(
+            'Please use a single expected output for '
+            '"// WebTestOutput=" in $filename');
     }
 
     if (contents.contains("@compile-error")) {
       isNegative = true;
     }
 
     if (contents.contains("@runtime-error") && hasRuntime) {
       isNegative = true;
     }
 
@@ skipping 15 matching lines @@
 
     return { "vmOptions": result,
              "dartOptions": dartOptions,
              "isNegative": isNegative,
              "isStaticClean" : isStaticClean,
              "otherScripts": otherScripts,
              "isMultitest": isMultitest,
              "containsLeadingHash": containsLeadingHash,
              "isolateStubs": isolateStubs,
              "containsDomImport": containsDomImport,
+             "webTestOutput": webTestOutput,
              "isLibraryDefinition": isLibraryDefinition,
              "containsSourceOrImport": containsSourceOrImport,
              "numStaticTypeAnnotations": numStaticTypeAnnotations,
              "numCompileTimeAnnotations": numCompileTimeAnnotations};
   }
 
+  static List<String> _readListOption(RegExp matcher, String contents) {
+    List<String> res = new List<String>();
+    for (var match in matcher.allMatches(contents)) {
+      res.addAll(match[1].split(' ').filter((e) => e != ''));
+    }
+    return res;
+  }
+
   List<List<String>> getVmOptions(Map optionsFromFile) {
     if (configuration['compiler'] == 'dart2js') {
       return [[]];
     } else {
       return optionsFromFile['vmOptions'];
     }
   }
 }
 
 
@@ skipping 373 matching lines @@
  * $noCrash tests are expected to be flaky but not crash
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
 """;
     print(report);
    }
 }