Update test.dart for detection output of machine formatted errors

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
@@ skipping 87 matching lines @@
   Function doDone;
   ReceivePort receiveTestName;
   TestExpectations testExpectations;
 
   CCTestSuite(Map this.configuration,
               String this.suiteName,
               String runnerName,
               List<String> this.statusFilePaths)
       : dartDir = TestUtils.dartDir() {
     runnerPath = TestUtils.buildDir(configuration) + '/' + runnerName;
-
   }
 
   void testNameHandler(String testName, ignore) {
     if (testName == "") {
       receiveTestName.close();
       doDone(true);
     } else {
       // Only run the tests that match the pattern. Use the name
       // "suiteName/testName" for cc tests.
       RegExp pattern = configuration['selectors'][suiteName];
@@ skipping 52 matching lines @@
 }
 
 
 class TestInformation {
   String filename;
   Map optionsFromFile;
   bool isNegative;
   bool isNegativeIfChecked;
   bool hasFatalTypeErrors;
   bool hasRuntimeErrors;
+  // expected outcome from multi-test  "static type error", "compile-time error", etc
+  String multitestOutcome;
 
   TestInformation(this.filename, this.optionsFromFile, this.isNegative,
                   this.isNegativeIfChecked, this.hasFatalTypeErrors,
-                  this.hasRuntimeErrors);
+                  this.hasRuntimeErrors, this.multitestOutcome);
 }
 
-
 /**
  * A standard [TestSuite] implementation that searches for tests in a
  * directory, and creates [TestCase]s that compile and/or run them.
  */
 class StandardTestSuite implements TestSuite {
   Map configuration;
   String suiteName;
   String directoryPath;
   List<String> statusFilePaths;
   Function doTest;
@@ skipping 133 matching lines @@
       case 'chromium':
       case 'frogium':
       case 'legium':
       case 'webdriver':
         enqueueBrowserTest(filename, testName, optionsFromFile,
                            expectations, isNegative);
         break;
       default:
         isNegative = isNegative ||
             (configuration['checked'] && info.isNegativeIfChecked);
-        bool enableFatalTypeErrors = false;
 
         if (configuration['component'] == 'dartc') {
-          // Only dartc supports fatal type errors. Enable fatal type
-          // errors with a flag and treat tests that have fatal type
-          // errors as negative.
-          // Also, tests that have runtime errors are not negative
-          // tests for dartc because dartc does not execute the test.
+          // dartc can detect static type errors by the
+          // format of the rror line

[Bill Hesse, 2012/03/02 08:55:18]

Typo - rror

           if (info.hasFatalTypeErrors) {
-            enableFatalTypeErrors = true;
             isNegative = true;
           } else if (info.hasRuntimeErrors) {
             isNegative = false;
           }
         }
 
         var argumentLists = argumentListsFromFile(filename,
-                                                  optionsFromFile,
-                                                  enableFatalTypeErrors);
+                                                  optionsFromFile);
 
         for (var args in argumentLists) {
           doTest(new TestCase('$suiteName/$testName',
                               [new Command(shellPath(), args)],
                               configuration,
                               completeHandler,
                               expectations,
-                              isNegative));
+                              isNegative,
+                              info));
         }
     }
   }
 
   Function makeTestCaseCreator(Map optionsFromFile) {
     return (String filename,
             bool isNegative,
             [bool isNegativeIfChecked = false,
              bool hasFatalTypeErrors = false,
-             bool hasRuntimeErrors = false]) {
+             bool hasRuntimeErrors = false,
+             String multitestOutcome = null]) {
       // Cache the test information for each test case.
       var info = new TestInformation(filename,
                                      optionsFromFile,
                                      isNegative,
                                      isNegativeIfChecked,
                                      hasFatalTypeErrors,
-                                     hasRuntimeErrors);
+                                     hasRuntimeErrors,
+                                     multitestOutcome);
       cachedTests.add(info);
       enqueueTestCaseFromTestInformation(info);
     };
   }
 
   void processFile(String filename) {
     if (!isTestFile(filename)) return;
 
     // Only run the tests that match the pattern.
     RegExp pattern = configuration['selectors'][suiteName];
@@ skipping 333 matching lines @@
     listingDone = true;
     if (activeTestGenerators == 0) {
       doDone();
     }
   }
 
   void completeHandler(TestCase testCase) {
   }
 
   List<List<String>> argumentListsFromFile(String filename,
-                                           Map optionsFromFile,
-                                           bool enableFatalTypeErrors) {
+                                           Map optionsFromFile) {
     List args = TestUtils.standardOptions(configuration);
     args.addAll(additionalOptions(filename));
-    if (enableFatalTypeErrors && configuration['component'] == 'dartc') {
-      args.add('--fatal-type-errors');
+    if (configuration['component'] == 'dartc') {
+      args.add('--error_format');
+      args.add('machine');
     }
 
     bool isMultitest = optionsFromFile["isMultitest"];
     List<String> dartOptions = optionsFromFile["dartOptions"];
     List<List<String>> vmOptionsList = optionsFromFile["vmOptions"];
     Expect.isTrue(!isMultitest || dartOptions == null);
     if (dartOptions == null) {
       args.add(filename);
     } else {
       var executable_name = dartOptions[0];
@@ skipping 16 matching lines @@
     }
 
     return result;
   }
 
   Map readOptionsFromFile(String filename) {
     RegExp testOptionsRegExp = const RegExp(@"// VMOptions=(.*)");
     RegExp dartOptionsRegExp = const RegExp(@"// DartOptions=(.*)");
     RegExp otherScriptsRegExp = const RegExp(@"// OtherScripts=(.*)");
     RegExp multiTestRegExp = const RegExp(@"/// [0-9][0-9]:(.*)");
+    RegExp staticTypeRegExp = const RegExp(@"/// ([0-9][0-9]:){0,1}\s*static type error");
+    RegExp compileTimeRegExp = const RegExp(@"/// ([0-9][0-9]:){0,1}\s*compile-time error");
+    RegExp staticCleanRegExp = const RegExp(@"// @static-clean");
     RegExp leadingHashRegExp = const RegExp(@"^#", multiLine: true);
     RegExp isolateStubsRegExp = const RegExp(@"// IsolateStubs=(.*)");
     RegExp domImportRegExp =
         const RegExp(@"^#import.*(dart:(dom|html)|html\.dart).*\)",
                      multiLine: true);
     RegExp libraryDefinitionRegExp =
         const RegExp(@"^#library\(", multiLine: true);
     RegExp sourceOrImportRegExp =
         const RegExp(@"^#(source|import)\(", multiLine: true);
 
     // Read the entire file into a byte buffer and transform it to a
     // String. This will treat the file as ascii but the only parts
     // we are interested in will be ascii in any case.
     RandomAccessFile file = new File(filename).openSync();
     List chars = new List(file.lengthSync());
     var offset = 0;
     while (offset != chars.length) {
       offset += file.readListSync(chars, offset, chars.length - offset);
     }
     file.closeSync();
     String contents = new String.fromCharCodes(chars);
     chars = null;
 
     // Find the options in the file.
     List<List> result = new List<List>();
     List<String> dartOptions;
     bool isNegative = false;
+    bool isStaticClean = false;
 
     Iterable<Match> matches = testOptionsRegExp.allMatches(contents);
     for (var match in matches) {
       result.add(match[1].split(' ').filter((e) => e != ''));
     }
     if (result.isEmpty()) result.add([]);
 
     matches = dartOptionsRegExp.allMatches(contents);
     for (var match in matches) {
       if (dartOptions != null) {
         throw new Exception(
             'More than one "// DartOptions=" line in test $filename');
       }
       dartOptions = match[1].split(' ').filter((e) => e != '');
     }
 
+    matches = staticCleanRegExp.allMatches(contents);

[Bill Hesse, 2012/03/02 08:55:18]

Is this overkill?  Why not just a single hasMatch?

[zundel, 2012/03/02 23:51:00]

I did that because I decided that multiple matches s/b fatal

+    for (var match in matches) {
+      if (isStaticClean) {
+        throw new Exception(
+            'More than one "// @static-clean=" line in test $filename');
+      }
+      isStaticClean = true;
+    }
+    
     List<String> otherScripts = new List<String>();
     matches = otherScriptsRegExp.allMatches(contents);
     for (var match in matches) {
       otherScripts.addAll(match[1].split(' ').filter((e) => e != ''));
     }
 
     if (contents.contains("@compile-error") ||
         contents.contains("@runtime-error")) {
       isNegative = true;
     }
-
+    

[Bill Hesse, 2012/03/02 08:55:18]

Whitespace.

     bool isMultitest = multiTestRegExp.hasMatch(contents);
     bool containsLeadingHash = leadingHashRegExp.hasMatch(contents);
     Match isolateMatch = isolateStubsRegExp.firstMatch(contents);
     String isolateStubs = isolateMatch != null ? isolateMatch[1] : '';
     bool containsDomImport = domImportRegExp.hasMatch(contents);
     bool isLibraryDefinition = libraryDefinitionRegExp.hasMatch(contents);
     bool containsSourceOrImport = sourceOrImportRegExp.hasMatch(contents);
-
+    int numStaticTypeAnnotations = 0;
+    for (var i in staticTypeRegExp.allMatches(contents)) {

[Bill Hesse, 2012/03/02 08:55:18]

Var dummy?
There is really no length field?  It is an iterator?

[zundel, 2012/03/02 23:51:00]

yes, its an Iterator.
http://api.dartlang.org/dart_core/RegExp.html

+      numStaticTypeAnnotations++;
+    }
+    int numCompileTimeAnnotations = 0;
+    for (var i in compileTimeRegExp.allMatches(contents)) {
+      numCompileTimeAnnotations++;
+    }
 
     return { "vmOptions": result,
              "dartOptions": dartOptions,
              "isNegative": isNegative,
+             "isStaticClean" : isStaticClean,
              "otherScripts": otherScripts,
              "isMultitest": isMultitest,
-             "containsLeadingHash" : containsLeadingHash,
-             "isolateStubs" : isolateStubs,
+             "containsLeadingHash": containsLeadingHash,
+             "isolateStubs": isolateStubs,
              "containsDomImport": containsDomImport,
              "isLibraryDefinition": isLibraryDefinition,
-             "containsSourceOrImport": containsSourceOrImport };
+             "containsSourceOrImport": containsSourceOrImport,
+             "numStaticTypeAnnotations": numStaticTypeAnnotations,
+             "numCompileTimeAnnotations": numCompileTimeAnnotations};
   }
 }
 
 
 class DartcCompilationTestSuite extends StandardTestSuite {
   List<String> _testDirs;
   int activityCount = 0;
 
   DartcCompilationTestSuite(Map configuration,
                             String suiteName,
@@ skipping 310 matching lines @@
  * $noCrash tests are expected to be flaky but not crash
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
 """;
     print(report);
    }
 }