Change test_runner.dart to always move on in the batch runner if the batch runner crashed.

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
@@ skipping 14 matching lines @@
 
   /** Command line arguments to the executable. */
   List<String> arguments;
 
   /** The actual command line that will be executed. */
   String commandLine;
 
   Command(this.executable, this.arguments) {
     commandLine = "$executable ${Strings.join(arguments, ' ')}";
   }
-  
+
   String toString() => commandLine;
 }
 
 /**
  * TestCase contains all the information needed to run a test and evaluate
  * its output.  Running a test involves starting a separate process, with
  * the executable and arguments given by the TestCase, and recording its
  * stdout and stderr output streams, and its exit code.  TestCase only
  * contains static information about the test; actually running the test is
  * performed by [ProcessQueue] using a [RunningProcess] object.
@@ skipping 121 matching lines @@
 
 /**
  * TestOutput records the output of a completed test: the process's exit code,
  * the standard output and standard error, whether the process timed out, and
  * the time the process took to run.  It also contains a pointer to the
  * [TestCase] this is the output of.
  */
 interface TestOutput default TestOutputImpl { 
   TestOutput.fromCase(TestCase testCase, int exitCode, bool timedOut,
     List<String> stdout, List<String> stderr, Duration time);
-  
+
   String get result();
-  
+
   bool get unexpectedOutput();
-  
+
   bool get hasCrashed();
-  
+
   bool get hasTimedOut(); 
-  
+
   bool get didFail();
-  
+
   List<String> get diagnostics();
 }
 
 class TestOutputImpl implements TestOutput {
   TestCase testCase;
   int exitCode;
   bool timedOut;
   bool failed = false;
   List<String> stdout;
   List<String> stderr;
   Duration time;
   List<String> diagnostics;
-  
+
   /**
    * Set to true if we encounter a condition in the output that indicates we
    * need to rerun this test.
    */
   bool requestRetry = false;
 
   // Don't call  this constructor, call TestOutput.fromCase() to 
   // get anew TestOutput instance.
   TestOutputImpl(this.testCase, this.exitCode, this.timedOut, this.stdout,
              this.stderr, this.time) {
     testCase.output = this;
     diagnostics = [];
   }
 
   factory TestOutputImpl.fromCase (testCase, exitCode, timedOut, stdout, stderr, 
     time) {
     if (testCase is BrowserTestCase) {
       return new BrowserTestOutputImpl(testCase, exitCode, timedOut,
         stdout, stderr, time);
     } else if (testCase.configuration['component'] == 'dartc') {
       return new AnalysisTestOutputImpl(testCase, exitCode, timedOut,
         stdout, stderr, time);
     }
     return new TestOutputImpl(testCase, exitCode, timedOut,
       stdout, stderr, time);
   }
-  
+
   String get result() =>
       hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS));
 
   bool get unexpectedOutput() => !testCase.expectedOutcomes.contains(result);
 
   bool get hasCrashed() {
     if (new Platform().operatingSystem() == 'windows') {
       // The VM uses std::abort to terminate on asserts.
       // std::abort terminates with exit code 3 on Windows.
       if (exitCode == 3) {
         return !timedOut;
       }
       return (!timedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0));
     }
     // The Java dartc runner exits with code 253 in case of unhandled
     // exceptions.
     return (!timedOut && ((exitCode < 0) || (exitCode == 253)));
   }
 
   bool get hasTimedOut() => timedOut;
 
   bool get didFail() {
     return (exitCode != 0 && !hasCrashed);
   }
-  
+
   // Reverse result of a negative test.
   bool get hasFailed() => (testCase.isNegative ? !didFail : didFail);
 
 }
 
 class BrowserTestOutputImpl extends TestOutputImpl {
   BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : 
     super(testCase, exitCode, timedOut, stdout, stderr, time);
-    
+
   bool get didFail() {
     // Browser case:
     // If the browser test failed, it may have been because DumpRenderTree
     // and the virtual framebuffer X server didn't hook up, or DRT crashed with
     // a core dump. Sometimes DRT crashes after it has set the stdout to PASS,
     // so we have to do this check first.
     for (String line in stderr) {
       if (line.contains('Gtk-WARNING **: cannot open display: :99') ||
         line.contains('Failed to run command. return code=1')) {
         // If we get the X server error, or DRT crashes with a core dump, retry
@@ skipping 20 matching lines @@
 
 // The static analyzer does not actaully execute code, so 
 // the criteria for success now depend on the text sent
 // to stderr.
 class AnalysisTestOutputImpl extends TestOutputImpl {
   boolean alreadyComputed = false;
   boolean failResult;
   AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : 
     super(testCase, exitCode, timedOut, stdout, stderr, time) {
   }
-  
+
   bool get didFail() {
     if (!alreadyComputed) {
       failResult = _didFail();
       alreadyComputed = true;
     }
     return failResult;
   }
-  
+
   bool _didFail() {
     if (hasCrashed) return false;
 
     List<String> errors = [];
     List<String> staticWarnings = [];
-    
+
     // Read the returned list of errors and stuff them away.
     for (String line in stderr) {
       if (line.length == 0) continue;
       List<String> fields = splitMachineError(line);
       if (fields[0] == 'ERROR') {
         errors.add(fields);
       } else if (fields[0] == 'WARNING') {
         // We only care about testing Static type warnings
         // ignore all others
         if (fields[1] == 'STATIC_TYPE') {
           staticWarnings.add(fields);
         } 
       }
       // OK to Skip error output that doesn't match the machine format
     }
     if (testCase.info != null 
         && testCase.info.optionsFromFile['isMultitest']) {
       return _didMultitestFail(errors, staticWarnings);
     }
     return _didStandardTestFail(errors, staticWarnings);
   }
-  
+
   bool _didMultitestFail(List errors, List staticWarnings) {
     String outcome = testCase.info.multitestOutcome;
     if ((outcome == '' || outcome == 'compile-time error') && errors.length > 0) {
       return true;
     } else if (outcome == 'static type error' && staticWarnings.length > 0) {
       return true;
     }
     return false;
   }
-  
+
   bool _didStandardTestFail(List errors, List staticWarnings) {
     bool hasFatalTypeErrors = false;
     int numStaticTypeAnnotations = 0;
     int numCompileTimeAnnotations = 0;
     var isStaticClean = false;
     if (testCase.info != null) {
       var optionsFromFile = testCase.info.optionsFromFile;
       hasFatalTypeErrors = optionsFromFile['hasFatalTypeErrors'];
       for (Command c in testCase.commands) {
         for (String arg in c.arguments) {
           if (arg == '--fatal-type-errors') {
             hasFatalTypeErrors = true;
             break;
           }
         }
       }
       numStaticTypeAnnotations = optionsFromFile['numStaticTypeAnnotations'];
       numCompileTimeAnnotations = optionsFromFile['numCompileTimeAnnotations'];
       isStaticClean = optionsFromFile['isStaticClean'];
     }
-    
+
     if (errors.length == 0) {
       if (!hasFatalTypeErrors && exitCode != 0) {
         diagnostics.add("EXIT CODE MISMATCH: Expected error message:");
         diagnostics.add("  command[0]:${testCase.commands[0]}");
         diagnostics.add("  exitCode:${exitCode}");
         return true;
       }
     } else if (exitCode == 0) {
       diagnostics.add("EXIT CODE MISMATCH: Unexpected error message:");
       diagnostics.add("  errors[0]:${errors[0]}");
       diagnostics.add("  command[0]:${testCase.commands[0]}");
       diagnostics.add("  exitCode:${exitCode}");
       return true;
     }
     if (numStaticTypeAnnotations > 0 && isStaticClean) {
       diagnostics.add("Cannot have both @static-clean and /// static type warning annotations.");
       return true;
     }
-    
+
     if (isStaticClean && staticWarnings.length > 0) {
       diagnostics.add("@static-clean annotation found but analyzer returned warnings.");
       return true;
     }
-    
+
     if (numCompileTimeAnnotations > 0 
         && numCompileTimeAnnotations < errors.length) {
-      
+
       // Expected compile-time errors were not returned.  The test did not 'fail' in the way
       // intended so don't return failed.
       diagnostics.add("Fewer compile time errors than annotated: ${numCompileTimeAnnotations}");
       return false;
     }
-    
+
     if (numStaticTypeAnnotations > 0 || hasFatalTypeErrors) {
       // TODO(zundel): match up the annotation line numbers
       // with the reported error line numbers
       if (staticWarnings.length < numStaticTypeAnnotations) {
         diagnostics.add("Fewer static type warnings than annotated: ${numStaticTypeAnnotations}");
         return true;
       }
       return false;
     } else if (errors.length != 0) {
       return true;
     } 
     return false;
   }
-  
+
   // Parse a line delimited by the | character using \ as an escape charager
   // like:  FOO|BAR|FOO\|BAR|FOO\\BAZ as 4 fields: FOO BAR FOO|BAR FOO\BAZ
   List<String> splitMachineError(String line) {
     StringBuffer field = new StringBuffer();
     List<String> result = [];
     bool escaped = false;
     for (var i = 0 ; i < line.length; i++) {
       var c = line[i];
       if (!escaped && c == '\\') {
         escaped = true;
@@ skipping 245 matching lines @@
 
   int _reportResult(String output) {
     _stdoutDrained = true;
     // output = '>>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}'
     var outcome = output.split(" ")[2];
     var exitCode = 0;
     if (outcome == "CRASH") exitCode = -10;
     if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1;
     new TestOutput.fromCase(_currentTest, exitCode, outcome == "TIMEOUT", 
                    _testStdout, _testStderr, new Date.now().difference(_startTime));
-    // Move on when both stdout and stderr has been drained.
-    if (_stderrDrained) _testCompleted();
+    // Move on when both stdout and stderr has been drained. If the test
+    // crashed, we restarted the process and therefore do not attempt to
+    // drain stderr.
+    if (_stderrDrained || (_currentTest.output.hasCrashed)) _testCompleted();
   }
 
   void _stderrDone() {
     _stderrDrained = true;
     // Move on when both stdout and stderr has been drained.
     if (_stdoutDrained) _testCompleted();

[zundel, 2012/03/08 12:22:16]

What about here?

[zundel, 2012/03/08 12:28:29]

nm, I guess we just wait for the process to exit and then _reportResult() will
be called from _exitHandler

   }
 
   Function _readStdout(StringInputStream stream, List<String> buffer) {
     return () {
       var status;
       var line = stream.readLine();
       while (line != null) {
         if (line.startsWith('>>> TEST')) {
           status = line;
         } else if (line.startsWith('>>> BATCH START')) {
@@ skipping 98 matching lines @@
   // configurations there is no need to repeatedly search the file
   // system, generate tests, and search test files for options.
   Map<String, List<TestInformation>> _testCache;
 
   /**
    * String indicating the browser used to run the tests. Empty if no browser
    * used.
    */
   String browserUsed = '';
 
-  /** 
+  /**
    * Process running the selenium server .jar (only used for Safari and Opera
    * tests.)
    */
   Process _seleniumServer = null;
 
   /** True if we are in the process of starting the server. */
   bool _startingServer = false;
 
   /** True if we find that there is already a selenium jar running. */
   bool _seleniumAlreadyRunning = false;
@@ skipping 122 matching lines @@
               }
             } else {
               print('\nDeletion of temp dir $_temporaryDirectory failed.');
             }
             _cleanupAndMarkDone();
           };
         }
       }
     }
   }
- 
-  /** 
+
+  /**
    * True if we are using a browser + platform combination that needs the
-   * Selenium server jar. 
+   * Selenium server jar.
    */
   bool get _needsSelenium() => new Platform().operatingSystem() == 'macos' &&
       browserUsed == 'safari';
 
   /** True if the Selenium Server is ready to be used. */
   bool get _isSeleniumAvailable() => _seleniumServer != null || 
       _seleniumAlreadyRunning;
 
-  /** 
+  /**
    * Restart all the processes that have been waiting/stopped for the server to 
    * start up. If we just call this once we end up with a single-"threaded" run.
    */
   void resumeTesting() {
     for (int i = 0; i < _maxProcesses; i++) _tryRunTest();
   }
 
   /** Start the Selenium Server jar, if appropriate for this platform. */
   void _ensureSeleniumServerRunning() {
     if (!_isSeleniumAvailable && !_startingServer) {
@@ skipping 28 matching lines @@
   void _runTest(TestCase test) {
     if (test.usesWebDriver) {
       browserUsed = test.configuration['browser'];
       if (_needsSelenium) _ensureSeleniumServerRunning();
     }
     _progress.testAdded();
     _tests.add(test);
     _tryRunTest();
   }
 
-  /** 
+  /**
    * Monitor the output of the Selenium server, to know when we are ready to
    * begin running tests.
    * source: Output(Stream) from the Java server.
    */
   Function makeSeleniumServerHandler(StringInputStream source) {
     return () {
       if (source.closed) return;  // TODO(whesse): Remove when bug is fixed.
       var line = source.readLine();
       while (null != line) {
         if (const RegExp(@".*Started.*Server.*").hasMatch(line) ||
             const RegExp(@"Exception.*Selenium is already running.*").hasMatch(
             line)) {
           resumeTesting();
         }
         line = source.readLine();
       }
     };
   }
 
-  /** 
+  /**
    * For browser tests using Safari or Opera, we need to use the Selenium 1.0
    * Java server.
    */
   void _startSeleniumServer() {
     // Get the absolute path to the Selenium jar.
     String filePath = new Options().script;
     String pathSep = new Platform().pathSeparator();
     int index = filePath.lastIndexOf(pathSep);
     filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}';
     var dir = new Directory(filePath);
@@ skipping 83 matching lines @@
         // the developer doesn't waste his or her time trying to fix a bunch of
         // tests that appear to be broken but were actually just flakes that
         // didn't get retried because there had already been one failure.
         bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
         new RunningProcess(test, allowRetry, this).start();
       }
       _numProcesses++;
     }
   }
 }