Update test.dart for detection output of machine formatted errors

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
@@ skipping 14 matching lines @@
 
   /** Command line arguments to the executable. */
   List<String> arguments;
 
   /** The actual command line that will be executed. */
   String commandLine;
 
   Command(this.executable, this.arguments) {
     commandLine = "$executable ${Strings.join(arguments, ' ')}";
   }
+  
+  String toString() => commandLine;
 }
 
 /**
  * TestCase contains all the information needed to run a test and evaluate
  * its output.  Running a test involves starting a separate process, with
  * the executable and arguments given by the TestCase, and recording its
  * stdout and stderr output streams, and its exit code.  TestCase only
  * contains static information about the test; actually running the test is
  * performed by [ProcessQueue] using a [RunningProcess] object.
  *
@@ skipping 13 matching lines @@
    * multiple sources that are run in isolation.
    */
   final List<Command> commands;
 
   Map configuration;
   String displayName;
   TestOutput output;
   bool isNegative;
   Set<String> expectedOutcomes;
   Function completedHandler;
+  TestInformation info;
 
   TestCase(this.displayName,
            this.commands,
            this.configuration,
            this.completedHandler,
            this.expectedOutcomes,
-           [this.isNegative = false]) {
+           [this.isNegative = false,
+            this.info = null]) {
     if (!isNegative) {
       this.isNegative = displayName.contains("NegativeTest");
     }
 
     // Special command handling. If a special command is specified
     // we have to completely rewrite the command that we are using.
     // We generate a new command-line that is the special command
     // where we replace '@' with the original command.
     var specialCommand = configuration['special-command'];
     if (!specialCommand.isEmpty()) {
@@ skipping 73 matching lines @@
       _lastArguments.getRange(1, _lastArguments.length - 1);
 }
 
 
 /**
  * TestOutput records the output of a completed test: the process's exit code,
  * the standard output and standard error, whether the process timed out, and
  * the time the process took to run.  It also contains a pointer to the
  * [TestCase] this is the output of.
  */
-class TestOutput {
+interface TestOutput default TestOutputImpl { 

[zundel, 2012/02/29 08:11:40]

I didn't come up with much better than what was here before, but I did change
this to an interface.  I didn't want to expose the generative constructor of
TestOutputImpl since I want to direct users of the API to the factory method
instead.

+  TestOutput.fromCase(TestCase testCase, int exitCode, bool timedOut,
+    List<String> stdout, List<String> stderr, Duration time);
+  
+  String get result();
+  
+  bool get unexpectedOutput();
+  
+  bool get hasCrashed();
+  
+  bool get hasTimedOut(); 
+  
+  bool get didFail();
+}
+
+class TestOutputImpl implements TestOutput {
   TestCase testCase;
   int exitCode;
   bool timedOut;
   bool failed = false;
   List<String> stdout;
   List<String> stderr;
   Duration time;
+  
   /**
    * Set to true if we encounter a condition in the output that indicates we
    * need to rerun this test.
    */
   bool requestRetry;
 
-  TestOutput(this.testCase, this.exitCode, this.timedOut, this.stdout,
+  // Don't call  this constructor, call TestOutput.fromCase() to 
+  // get anew TestOutput instance.
+  TestOutputImpl(this.testCase, this.exitCode, this.timedOut, this.stdout,
              this.stderr, this.time) {
     testCase.output = this;
     requestRetry = false;
   }
 
+  factory TestOutputImpl.fromCase (testCase, exitCode, timedOut, stdout, stderr, 
+    time) {
+    if (testCase is BrowserTestCase) {
+      return new BrowserTestOutputImpl(testCase, exitCode, timedOut,
+        stdout, stderr, time);
+    } else if (testCase.configuration['component'] == 'dartc') {
+      return new AnalysisTestOutputImpl(testCase, exitCode, timedOut,
+        stdout, stderr, time);
+    }
+    return new TestOutputImpl(testCase, exitCode, timedOut,
+      stdout, stderr, time);
+  }
+  
   String get result() =>
       hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS));
 
   bool get unexpectedOutput() => !testCase.expectedOutcomes.contains(result);
 
   bool get hasCrashed() {
     if (new Platform().operatingSystem() == 'windows') {
       // The VM uses std::abort to terminate on asserts.
       // std::abort terminates with exit code 3 on Windows.
       if (exitCode == 3) {
         return !timedOut;
       }
       return (!timedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0));
     }
     // The Java dartc runner exits with code 253 in case of unhandled
     // exceptions.
     return (!timedOut && ((exitCode < 0) || (exitCode == 253)));
   }
 
   bool get hasTimedOut() => timedOut;
 
   bool get didFail() {
-    if (testCase is !BrowserTestCase) return (exitCode != 0 && !hasCrashed);
+    return (exitCode != 0 && !hasCrashed);
+  }
+  
+  // Reverse result of a negative test.
+  bool get hasFailed() => (testCase.isNegative ? !didFail : didFail);  
+}
 
+class BrowserTestOutputImpl extends TestOutputImpl {
+  BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : 
+    super(testCase, exitCode, timedOut, stdout, stderr, time);
+    
+  bool get didFail() {
     // Browser case:
     // If the browser test failed, it may have been because DumpRenderTree
     // and the virtual framebuffer X server didn't hook up, or DRT crashed with
     // a core dump. Sometimes DRT crashes after it has set the stdout to PASS,
     // so we have to do this check first.
     for (String line in stderr) {
       if (line.contains('Gtk-WARNING **: cannot open display: :99') ||
         line.contains('Failed to run command. return code=1')) {
         // If we get the X server error, or DRT crashes with a core dump, retry
         // the test.
         if (testCase.dynamic.numRetries > 0) {
           requestRetry = true;
         }
         return true;
       }
     }
 
     // Browser tests fail unless stdout contains
     // 'Content-Type: text/plain\nPASS'.
     String previous_line = '';
     for (String line in stdout) {
       if (line == 'PASS' && previous_line == 'Content-Type: text/plain') {
         return (exitCode != 0 && !hasCrashed);
       }
       previous_line = line;
     }
-
     return true;
   }
+}
 
-  // Reverse result of a negative test.
-  bool get hasFailed() => (testCase.isNegative ? !didFail : didFail);
+// The static analyzer does not actaully execute code, so 
+// the criteria for success now depend on the text sent
+// to stderr.
+class AnalysisTestOutputImpl extends TestOutputImpl { 
+  AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : 
+    super(testCase, exitCode, timedOut, stdout, stderr, time) {
+  }
+  
+  bool get didFail() {
+    if (hasCrashed) return false;
+
+    List<String> errors = [];
+    List<String> staticWarnings = [];
+    
+    // Read the returned list of errors and stuff them away.
+    for (String line in stderr) {
+      if (line.length == 0) continue;
+      List<String> fields = splitMachineError(line);
+      switch(fields[0]) {
+      case 'ERROR':
+        errors.add(fields);
+        break;
+      case 'WARNING':
+        // We only care about testing Static type warnings
+        // ignore all others
+        if (fields[1] == 'STATIC_TYPE') {
+          staticWarnings.add(fields);
+        } 
+        break;
+      default:
+        // Skip error output that doesn't match the machine format
+      }
+    }
+    if (testCase.info != null 
+        && testCase.info.optionsFromFile['isMultitest']) {
+      return _didMultitestFail(errors, staticWarnings);
+    }
+    return _didStandardTestFail(errors, staticWarnings);
+  }
+  
+  bool _didMultitestFail(List errors, List staticWarnings) {
+    String outcome = testCase.info.multitestOutcome;
+    if ((outcome == '' || outcome == 'compile-time error') && errors.length > 0) {
+      return true;
+    } else if (outcome == 'static type error' && staticWarnings.length > 0) {
+      return true;
+    }
+    return false;
+  }
+  
+  bool _didStandardTestFail(List errors, List staticWarnings) {
+    bool hasFatalTypeErrors = false;
+    int numStaticTypeAnnotations = 0;
+    int numCompileTimeAnnotations = 0;
+    if (testCase.info != null) {
+      var optionsFromFile = testCase.info.optionsFromFile;
+      hasFatalTypeErrors = optionsFromFile['hasFatalTypeErrors'];
+      for (Command c in testCase.commands) {
+        for (String arg in c.arguments) {
+          if (arg == '--fatal-type-errors') {
+            hasFatalTypeErrors = true;
+            break;
+          }
+        }
+      }
+      numStaticTypeAnnotations = optionsFromFile['numStaticTypeAnnotations'];
+      numCompileTimeAnnotations = optionsFromFile['numCompileTimeAnnotations'];
+    }
+    
+    // TODO(zundel): These assertions are catching some sort of issue 
+    // where the output between two test cases is getting crossed.
+    if (errors.length == 0) {
+      if (!hasFatalTypeErrors) {
+        Expect.isTrue(exitCode == 0,
+            "Expected error: exitCode:${exitCode} command[0]:${testCase.commands[0]}");
+      }
+    } else {
+      Expect.isTrue(exitCode != 0,
+        "Unexpected error: exitCode:${exitCode} command[0]:${testCase.commands[0]}  errors[0]:${errors[0]}");
+    }
+    
+    if (numCompileTimeAnnotations > 0 
+        && numCompileTimeAnnotations < errors.length) {
+      // Expected compile-time errors were not returned.  The test did not 'fail' in the way
+      // intended so don't return failed.
+      // TODO(zundel): give a good diagnostic message here
+      return false;
+    }
+    
+    if (numStaticTypeAnnotations > 0 || hasFatalTypeErrors) {
+      // TODO(zundel): match up the annotation line numbers
+      // with the reported error line numbers
+      if (staticWarnings.length < numStaticTypeAnnotations) {
+        // TODO(zundel): How to give a good diagnostic message here?
+        return true;
+      }
+      return false;
+    } else if (errors.length != 0) {
+      return true;
+    } 
+    return false;
+  }
+  
+  // Parse a line delimited by the | character using \ as an escape charager
+  // like:  FOO|BAR|FOO\|BAR|FOO\\BAZ as 4 fields: FOO BAR FOO|BAR FOO\BAZ
+  List<String> splitMachineError(String line) {
+    StringBuffer field = new StringBuffer();
+    List<String> result = [];
+    bool escaped = false;
+    for (var i = 0 ; i < line.length; i++) {
+      var c = line[i];
+      if (!escaped && c == '\\') {
+        escaped = true;
+        continue;
+      }
+      escaped = false;
+      if (c == '|') {
+        result.add(field.toString());
+        field.clear();
+        continue;
+      }
+      field.add(c);
+    }
+    result.add(field.toString());
+    return result;
+  }
 }
 
 /**
  * A RunningProcess actually runs a test, getting the command lines from
  * its [TestCase], starting the test process (and first, a compilation
  * process if the TestCase is a [BrowserTestCase]), creating a timeout
  * timer, and recording the results in a new [TestOutput] object, which it
  * attaches to the TestCase.  The lifetime of the RunningProcess is limited
  * to the time it takes to start the process, run the process, and record
  * the result; there are no pointers to it, so it should be available to
@@ skipping 16 matching lines @@
 
   RunningProcess(TestCase this.testCase,
       [this.allowRetries, this.processQueue]);
 
   /**
    * Called when all commands are executed. [exitCode] is 0 if all command
    * succeded, otherwise it will have the exit code of the first failing
    * command.
    */
   void testComplete(int exitCode) {
-    new TestOutput(testCase, exitCode, timedOut, stdout,
-                   stderr, new Date.now().difference(startTime));
+    new TestOutput.fromCase(testCase, exitCode, timedOut, stdout,
+                            stderr, new Date.now().difference(startTime));
     timeoutTimer.cancel();
     if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) {
       print(testCase.displayName);
       for (var line in testCase.output.stderr) print(line);
       for (var line in testCase.output.stdout) print(line);
     }
     if (allowRetries != null && allowRetries
         && testCase.configuration['component'] == 'webdriver' &&
         testCase.output.unexpectedOutput && testCase.numRetries > 0) {
       // Selenium tests can be flaky. Try rerunning.
       testCase.output.requestRetry = true;
     }
     if (testCase.output.requestRetry) {
       testCase.output.requestRetry = false;
       this.timedOut = false;
       testCase.dynamic.numRetries--;
-      print("Potential flake. " +
-          "Re-running ${testCase.displayName} " +
+      print("Potential flake. Re-running ${testCase.displayName} " +
           "(${testCase.dynamic.numRetries} attempt(s) remains)");
       this.start();
     } else {
       testCase.completed();
     }
   }
 
   /**
    * Process exit handler called at the end of every command. It internally
    * treats all but the last command as compilation steps. The last command is
@@ skipping 135 matching lines @@
         bool shutdownMillisecs = 30000;
         new Timer((e) { if (!closed) _process.kill(); }, shutdownMillisecs);
       } else {
         _process.kill();
       }
     }
   }
 
   void doStartTest(TestCase testCase) {
     _startTime = new Date.now();
-    _testStdout = new List<String>();
-    _testStderr = new List<String>();
+    _testStdout = [];
+    _testStderr = [];
     _stdoutStream.lineHandler = _readOutput(_stdoutStream, _testStdout);
     _stderrStream.lineHandler = _readOutput(_stderrStream, _testStderr);
     _timer = new Timer(_timeoutHandler, testCase.timeout * 1000);
     var line = _createArgumentsLine(testCase.batchTestArguments);
     _process.stdin.write(line.charCodes());
   }
 
   String _createArgumentsLine(List<String> arguments) {
     return Strings.join(arguments, ' ') + '\n';
   }
 
   int _reportResult(String output) {
     var test = _currentTest;
     _currentTest = null;
 
     // output = '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}'
     var outcome = output.split(" ")[2];
     var exitCode = 0;
     if (outcome == "CRASH") exitCode = -10;
     if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1;
-    new TestOutput(test, exitCode, outcome == "TIMEOUT", _testStdout,
-                   _testStderr, new Date.now().difference(_startTime));
+    new TestOutput.fromCase(test, exitCode, outcome == "TIMEOUT", _testStdout,
+                            _testStderr, new Date.now().difference(_startTime));
+    _testStdout = [];
+    _testStderr = [];
     test.completed();
   }
 
   Function _readOutput(StringInputStream stream, List<String> buffer) {
     return () {
       var status;
       var line = stream.readLine();
       // Drain the input stream to get the error output.
       while (line != null) {
         if (line.startsWith('>>> TEST')) {
@@ skipping 12 matching lines @@
         // For crashing processes, let the exit handler deal with it.
         if (!status.contains("CRASH")) {
           _reportResult(status);
         }
       }
     };
   }
 
   void _exitHandler(exitCode) {
     if (_timer != null) _timer.cancel();
+    _reportResult(">>> TEST CRASH");

[zundel, 2012/02/29 08:11:40]

I moved this because _startProcess() clears the _testStdout and _testStdin
fields, thus we loose diagnostic output on a crash.

     _process.close();
-    _startProcess(() {
-      _reportResult(">>> TEST CRASH");
-    });
+    _startProcess(() {});
   }
 
   void _timeoutHandler(ignore) {
     _process.exitHandler = (exitCode) {
+      _reportResult(">>> TEST TIMEOUT");
       _process.close();
-      _startProcess(() {
-        _reportResult(">>> TEST TIMEOUT");
-      });
+      _startProcess(() {});
     };
     _process.kill();
   }
 
   void _startProcess(then) {
     _process = new Process.start(_executable, _batchArguments);
     _stdoutStream = new StringInputStream(_process.stdout);
     _stderrStream = new StringInputStream(_process.stderr);
     _testStdout = new List<String>();
     _testStderr = new List<String>();
@@ skipping 242 matching lines @@
         // the developer doesn't waste his or her time trying to fix a bunch of
         // tests that appear to be broken but were actually just flakes that
         // didn't get retried because there had already been one failure.
         bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
         new RunningProcess(test, allowRetry, this).start();
       }
       _numProcesses++;
     }
   }
 }