Reduce flakiness of browser tests.

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #library("test_runner");
 
 #import("dart:io");
 #import("status_file_parser.dart");
 #import("test_progress.dart");
 #import("test_suite.dart");
@@ skipping 78 matching lines @@
 
 
 /**
  * BrowserTestCase has an extra compilation command that is run by
  * RunningProcess.start(), and it checks conditions on the test output
  * in TestOutput.didFail().
  */
 class BrowserTestCase extends TestCase {
   String compilerPath;
   List<String> compilerArguments;
+  /** 
+   * Indicates if this test is a rerun, to compensate for flaky browser tests. 
+   */
+  bool isRerun;
 
   BrowserTestCase(displayName,
                     this.compilerPath,
                     this.compilerArguments,
                     executablePath,
                     arguments,
                     configuration,
                     completedHandler,
                     expectedOutcomes,
                     [isNegative = false]) : super(displayName,
                                                   executablePath,
                                                   arguments,
                                                   configuration,
                                                   completedHandler,
                                                   expectedOutcomes,
                                                   isNegative) {
     if (compilerPath != null) {
       commandLine = 'execution command: $commandLine';
       String compilationCommand =
           '$compilerPath ${Strings.join(compilerArguments, " ")}';
       commandLine = 'compilation command: $compilationCommand\n$commandLine';
     }
+    isRerun = false;
   }
 }
 
 
 class TestOutput {
   // The TestCase this is the output from.
   TestCase testCase;
   int exitCode;
   bool timedOut;
   bool failed = false;
@@ skipping 68 matching lines @@
   List<String> stderr;
   List<Function> handlers;
 
   RunningProcess(this.testCase);
 
   void exitHandler(int exitCode) {
     new TestOutput(testCase, exitCode, timedOut, stdout,
                    stderr, new Date.now().difference(startTime));
     process.close();
     timeoutTimer.cancel();
-    testCase.completed();
+    if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) {
+      print(testCase.output.stdout);
+      print(testCase.output.stderr);
+    }
+    if (testCase is BrowserTestCase && testCase.output.unexpectedOutput && 
+        !testCase.isRerun) {
+      // Selenium tests can be flaky. Try rerunning.
+      testCase.isRerun = true;

[Bill Hesse, 2012/02/08 18:15:06]

Can we print a message in this case?  Is this the best solution for rerunning
tests?  We need the capability to rerun tests for the DumpRenderTree virtual X
server failures - can we add a "request retry" flag to TestOutput, and use that
flag here, so we can request retries in more circumstances?  I think that if we
allow rerunning tests, we should fix the flaky DRT at the same time.

+      this.timedOut = false;
+      this.start();
+    } else {
+      testCase.completed();
+    }
   }
 
   void compilerExitHandler(int exitCode) {
     if (exitCode != 0) {
       stderr.add('test.dart: Compilation step failed (exit code $exitCode)\n');
       exitHandler(exitCode);
     } else {
       process.close();
       stderr.add('test.dart: Compilation finished, starting execution\n');
       stdout.add('test.dart: Compilation finished, starting execution\n');
@@ skipping 340 matching lines @@
           test.displayName != 'dartc/junit_tests') {
         _ensureDartcBatchRunnersStarted(test.executablePath);
         _getDartcBatchRunnerProcess().startTest(test);
       } else {
         new RunningProcess(test).start();
       }
       _numProcesses++;
     }
   }
 }