Webdriver install script, take 2.

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
@@ skipping 114 matching lines @@
     final component = configuration['component'];
     final mode = configuration['mode'];
     final arch = configuration['arch'];
     return "$component ${mode}_$arch";
   }
 
   List<String> get batchRunnerArguments() => ['-batch'];
   List<String> get batchTestArguments() => commands.last().arguments;
 
   void completed() { completedHandler(this); }
+
+  bool get usesWebDriver() => configuration['component'] == 'webdriver';
 }
 
 
 /**
  * BrowserTestCase has an extra compilation command that is run in a separate
  * process, before the regular test is run as in the base class [TestCase].
  * If the compilation command fails, then the rest of the test is not run.
  */
 class BrowserTestCase extends TestCase {
   /**
@@ skipping 328 matching lines @@
   void testComplete(int exitCode) {
     new TestOutput.fromCase(testCase, exitCode, timedOut, stdout,
                             stderr, new Date.now().difference(startTime));
     timeoutTimer.cancel();
     if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) {
       print(testCase.displayName);
       for (var line in testCase.output.stderr) print(line);
       for (var line in testCase.output.stdout) print(line);
     }
     if (allowRetries != null && allowRetries
-        && testCase.configuration['component'] == 'webdriver' &&
-        testCase.output.unexpectedOutput && testCase.numRetries > 0) {
+        && testCase.usesWebDriver && testCase.output.unexpectedOutput 
+        && testCase.numRetries > 0) {
       // Selenium tests can be flaky. Try rerunning.
       testCase.output.requestRetry = true;
     }
     if (testCase.output.requestRetry) {
       testCase.output.requestRetry = false;
       this.timedOut = false;
       testCase.dynamic.numRetries--;
       print("Potential flake. Re-running ${testCase.displayName} " +
           "(${testCase.dynamic.numRetries} attempt(s) remains)");
       this.start();
@@ skipping 12 matching lines @@
     int totalSteps = testCase.commands.length;
     String suffix =' (step $currentStep of $totalSteps)';
     if (currentStep == totalSteps) { // done with test command
       testComplete(exitCode);
     } else if (exitCode != 0) {
       stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n');
       testComplete(exitCode);
     } else {
       stderr.add('test.dart: Compilion finished $suffix\n');
       stdout.add('test.dart: Compilion finished $suffix\n');
-      if (currentStep == totalSteps - 1
-          && testCase.configuration['component'] == 'webdriver') {
+      if (currentStep == totalSteps - 1 && testCase.usesWebDriver &&
+          !testCase.configuration['noBatch']) {
         // Note: processQueue will always be non-null for component == webdriver
         // (It is only null for component == vm)
         processQueue._getBatchRunner(testCase).startTest(testCase);
       } else {
         runCommand(testCase.commands[currentStep++], stepExitHandler);
       }
     }
   }
 
   Function makeReadHandler(StringInputStream source, List<String> destination) {
@@ skipping 57 matching lines @@
   bool _stdoutDrained = false;
   bool _stderrDrained = false;
   Date _startTime;
   Timer _timer;
 
   bool _isWebDriver;
 
   BatchRunnerProcess(TestCase testCase) {
     _executable = testCase.commands.last().executable;
     _batchArguments = testCase.batchRunnerArguments;
-    _isWebDriver = testCase.configuration['component'] == 'webdriver';
+    _isWebDriver = testCase.usesWebDriver;
   }
 
   bool get active() => _currentTest != null;
 
   void startTest(TestCase testCase) {
     _currentTest = testCase;
     if (_process === null) {
       // Start process if not yet started.
       _executable = testCase.commands.last().executable;
       _startProcess(() {
@@ skipping 164 matching lines @@
  *
  * Because multiple configurations may be run on each test suite, the
  * ProcessQueue contains a cache in which a test suite may record information
  * about its list of tests, and may retrieve that information when it is called
  * upon to enqueue its tests again.
  */
 class ProcessQueue {
   int _numProcesses = 0;
   int _activeTestListers = 0;
   int _maxProcesses;
+
   /** The number of tests we allow to actually fail before we stop retrying. */
   int _MAX_FAILED_NO_RETRY = 4;
   bool _verbose;
   bool _listTests;
   bool _keepGeneratedTests;
   Function _enqueueMoreWork;
   Queue<TestCase> _tests;
   ProgressIndicator _progress;
   String _temporaryDirectory;
+
   // For dartc/selenium batch processing we keep a list of batch processes.
   Map<String, List<BatchRunnerProcess>> _batchProcesses;
 
   // Cache information about test cases per test suite. For multiple
   // configurations there is no need to repeatedly search the file
   // system, generate tests, and search test files for options.
   Map<String, List<TestInformation>> _testCache;
+
   /**
    * String indicating the browser used to run the tests. Empty if no browser
    * used.
    */
-  String browserUsed;
+  String browserUsed = '';
+
+  /** 
+   * Process running the selenium server .jar (only used for Safari and Opera
+   * tests.)
+   */
+  Process _seleniumServer = null;
+
+  /** True if we are in the process of starting the server. */
+  bool _startingServer = false;
+
+  /** True if we find that there is already a selenium jar running. */
+  bool _seleniumAlreadyRunning = false;
 
   ProcessQueue(int this._maxProcesses,
                String progress,
                Date startTime,
                bool printTiming,
                Function this._enqueueMoreWork,
                [bool this._verbose = false,
                 bool this._listTests = false,
                 bool this._keepGeneratedTests = false])
       : _tests = new Queue<TestCase>(),
         _progress = new ProgressIndicator.fromName(progress,
                                                    startTime,
                                                    printTiming),
         _batchProcesses = new Map<String, List<BatchRunnerProcess>>(),
         _testCache = new Map<String, List<TestInformation>>() {
     if (!_enqueueMoreWork(this)) _progress.allDone();
-    browserUsed = '';
   }
 
   /**
    * Registers a TestSuite so that all of its tests will be run.
    */
   void addTestSuite(TestSuite testSuite) {
     _activeTestListers++;
     testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory,
                           _testListerDone);
   }
@@ skipping 55 matching lines @@
     }
   }
 
   /**
    * Perform any cleanup needed once all tests in a TestSuite have completed
    * and notify our progress indicator that we are done.
    */
   void _cleanupAndMarkDone() {
     if (browserUsed != '') {
       killZombieBrowsers();
+      if (_seleniumServer != null) {
+        _seleniumServer.kill();
+      }
     } else {
       _progress.allDone();
     }
   }
 
   void _checkDone() {
     // When there are no more active test listers ask for more work
     // from process queue users.
     if (_activeTestListers == 0 && !_enqueueMoreWork(this)) {
       _progress.allTestsKnown();
@@ skipping 18 matching lines @@
               }
             } else {
               print('\nDeletion of temp dir $_temporaryDirectory failed.');
             }
             _cleanupAndMarkDone();
           };
         }
       }
     }
   }
+ 
+  /** 
+   * True if we are using a browser + platform combination that needs the
+   * Selenium server jar. 
+   */
+  bool get _needsSelenium() => new Platform().operatingSystem() == 'macos' &&
+      browserUsed == 'safari';
+
+  /** True if the Selenium Server is ready to be used. */
+  bool get _isSeleniumAvailable() => _seleniumServer != null || 
+      _seleniumAlreadyRunning;
+
+  /** 
+   * Restart all the processes that have been waiting/stopped for the server to 
+   * start up. If we just call this once we end up with a single-"threaded" run.
+   */
+  void resumeTesting() {
+    for (int i = 0; i < _maxProcesses; i++) _tryRunTest();
+  }
+
+  /** Start the Selenium Server jar, if appropriate for this platform. */
+  void _ensureSeleniumServerRunning() {
+    if (!_isSeleniumAvailable && !_startingServer) {
+      _startingServer = true;
+
+      // Check to see if the jar was already running before the program started.
+      String cmd = 'ps';
+      var arg = ['aux'];
+      if (new Platform().operatingSystem() == 'windows') {
+        cmd = 'tasklist';
+        arg.add('/v');
+      }
+      Process p = new Process.start(cmd, arg);
+      final StringInputStream stdoutStringStream = 
+          new StringInputStream(p.stdout);
+      stdoutStringStream.onLine = () {
+        var line = stdoutStringStream.readLine();
+        while (null != line) {
+          if (const RegExp(@".*selenium-server-standalone.*").hasMatch(line)) {
+            _seleniumAlreadyRunning = true;
+            resumeTesting();
+          }
+          line = stdoutStringStream.readLine();
+        }
+        if (!_isSeleniumAvailable) {
+          _startSeleniumServer();
+        }
+      };
+    }
+  }
 
   void _runTest(TestCase test) {
-    if (test.configuration['component'] == 'webdriver') {
+    if (test.usesWebDriver) {
       browserUsed = test.configuration['browser'];
+      if (_needsSelenium) _ensureSeleniumServerRunning();
     }
     _progress.testAdded();
     _tests.add(test);
     _tryRunTest();
   }
 
+  /** 
+   * Monitor the output of the Selenium server, to know when we are ready to
+   * begin running tests.
+   * source: Output(Stream) from the Java server.
+   */
+  Function makeSeleniumServerHandler(StringInputStream source) {
+    return () {
+      if (source.closed) return;  // TODO(whesse): Remove when bug is fixed.
+      var line = source.readLine();
+      while (null != line) {
+        if (const RegExp(@".*Started.*Server.*").hasMatch(line) ||
+            const RegExp(@"Exception.*Selenium is already running.*").hasMatch(
+            line)) {
+          resumeTesting();
+        }
+        line = source.readLine();
+      }
+    };
+  }
+
+  /** 
+   * For browser tests using Safari or Opera, we need to use the Selenium 1.0
+   * Java server.
+   */
+  void _startSeleniumServer() {
+    // Get the absolute path to the Selenium jar.
+    String filePath = new Options().script;
+    String pathSep = new Platform().pathSeparator();
+    int index = filePath.lastIndexOf(pathSep);
+    filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}';
+    var dir = new Directory(filePath);
+    dir.onFile = (String file) {
+      if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file)
+          && _seleniumServer == null) {
+        _seleniumServer = new Process.start('java', ['-jar', file]);
+        // Heads up: there seems to an obscure data race of some form in
+        // the VM between launching the server process and launching the test
+        // tasks that disappears when you read IO (which is convenient, since
+        // that is our condition for knowing that the server is ready). 
+        StringInputStream stdoutStringStream =
+            new StringInputStream(_seleniumServer.stdout);
+        StringInputStream stderrStringStream =
+            new StringInputStream(_seleniumServer.stderr);
+        stdoutStringStream.onLine =
+            makeSeleniumServerHandler(stdoutStringStream);
+        stderrStringStream.onLine =
+            makeSeleniumServerHandler(stderrStringStream);
+      }
+    };
+    dir.list();
+  }
+
   void _terminateBatchRunners() {
     for (var runners in _batchProcesses.getValues()) {
       for (var runner in runners) {
         runner.terminate();
       }
     }
   }
 
   BatchRunnerProcess _getBatchRunner(TestCase test) {
     // Start batch processes if needed
@@ skipping 10 matching lines @@
     for (var runner in runners) {
       if (!runner.active) return runner;
     }
     throw new Exception('Unable to find inactive batch runner.');
   }
 
   void _tryRunTest() {
     _checkDone();
     if (_numProcesses < _maxProcesses && !_tests.isEmpty()) {
       TestCase test = _tests.removeFirst();
-      if (_verbose) print(test.commands.last().commandLine);
       if (_listTests) {
         final String tab = '\t';
         String outcomes =
             Strings.join(new List.from(test.expectedOutcomes), ',');
         print(test.displayName + tab + outcomes + tab + test.isNegative +
               tab + Strings.join(test.commands.last().arguments, tab));
         return;
       }
+      if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) {
+        // The server is not ready to run Selenium tests. Put the test back in
+        // the queue.
+        _tests.addFirst(test);
+        return;
+      }
+      if (_verbose) print(test.commands.last().commandLine);
       _progress.start(test);
       Function oldCallback = test.completedHandler;
       Function wrapper = (TestCase test_arg) {
         _numProcesses--;
         _progress.done(test_arg);
         _tryRunTest();
         oldCallback(test_arg);
       };
       test.completedHandler = wrapper;
       if (test.configuration['component'] == 'dartc' &&
           test.displayName != 'dartc/junit_tests') {
         _getBatchRunner(test).startTest(test);
       } else {
         // Once we've actually failed a test, technically, we wouldn't need to
         // bother retrying any subsequent tests since the bot is already red.
         // However, we continue to retry tests until we have actually failed
         // four tests (arbitrarily chosen) for more debugable output, so that
         // the developer doesn't waste his or her time trying to fix a bunch of
         // tests that appear to be broken but were actually just flakes that
         // didn't get retried because there had already been one failure.
         bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
         new RunningProcess(test, allowRetry, this).start();
       }
       _numProcesses++;
     }
   }
 }