Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(60)

Side by Side Diff: tools/testing/dart/test_runner.dart

Issue 9572007: Revert "Adding webdriver setup script." (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /** 5 /**
6 * Classes and methods for executing tests. 6 * Classes and methods for executing tests.
7 * 7 *
8 * This module includes: 8 * This module includes:
9 * - Managing parallel execution of tests, including timeout checks. 9 * - Managing parallel execution of tests, including timeout checks.
10 * - Evaluating the output of each test as pass/fail/crash/timeout. 10 * - Evaluating the output of each test as pass/fail/crash/timeout.
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after
121 final component = configuration['component']; 121 final component = configuration['component'];
122 final mode = configuration['mode']; 122 final mode = configuration['mode'];
123 final arch = configuration['arch']; 123 final arch = configuration['arch'];
124 return "$component ${mode}_$arch"; 124 return "$component ${mode}_$arch";
125 } 125 }
126 126
127 List<String> get batchRunnerArguments() => ['-batch']; 127 List<String> get batchRunnerArguments() => ['-batch'];
128 List<String> get batchTestArguments() => commands.last().arguments; 128 List<String> get batchTestArguments() => commands.last().arguments;
129 129
130 void completed() { completedHandler(this); } 130 void completed() { completedHandler(this); }
131
132 bool get usesWebDriver() => configuration['component'] == 'webdriver';
133 } 131 }
134 132
135 133
136 /** 134 /**
137 * BrowserTestCase has an extra compilation command that is run in a separate 135 * BrowserTestCase has an extra compilation command that is run in a separate
138 * process, before the regular test is run as in the base class [TestCase]. 136 * process, before the regular test is run as in the base class [TestCase].
139 * If the compilation command fails, then the rest of the test is not run. 137 * If the compilation command fails, then the rest of the test is not run.
140 */ 138 */
141 class BrowserTestCase extends TestCase { 139 class BrowserTestCase extends TestCase {
142 /** 140 /**
(...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after
281 void testComplete(int exitCode) { 279 void testComplete(int exitCode) {
282 new TestOutput(testCase, exitCode, timedOut, stdout, 280 new TestOutput(testCase, exitCode, timedOut, stdout,
283 stderr, new Date.now().difference(startTime)); 281 stderr, new Date.now().difference(startTime));
284 timeoutTimer.cancel(); 282 timeoutTimer.cancel();
285 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { 283 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) {
286 print(testCase.displayName); 284 print(testCase.displayName);
287 for (var line in testCase.output.stderr) print(line); 285 for (var line in testCase.output.stderr) print(line);
288 for (var line in testCase.output.stdout) print(line); 286 for (var line in testCase.output.stdout) print(line);
289 } 287 }
290 if (allowRetries != null && allowRetries 288 if (allowRetries != null && allowRetries
291 && testCase.usesWebDriver && testCase.output.unexpectedOutput 289 && testCase.configuration['component'] == 'webdriver' &&
292 && testCase.numRetries > 0) { 290 testCase.output.unexpectedOutput && testCase.numRetries > 0) {
293 // Selenium tests can be flaky. Try rerunning. 291 // Selenium tests can be flaky. Try rerunning.
294 testCase.output.requestRetry = true; 292 testCase.output.requestRetry = true;
295 } 293 }
296 if (testCase.output.requestRetry) { 294 if (testCase.output.requestRetry) {
297 testCase.output.requestRetry = false; 295 testCase.output.requestRetry = false;
298 this.timedOut = false; 296 this.timedOut = false;
299 testCase.dynamic.numRetries--; 297 testCase.dynamic.numRetries--;
300 print("Potential flake. " + 298 print("Potential flake. " +
301 "Re-running ${testCase.displayName} " + 299 "Re-running ${testCase.displayName} " +
302 "(${testCase.dynamic.numRetries} attempt(s) remains)"); 300 "(${testCase.dynamic.numRetries} attempt(s) remains)");
(...skipping 13 matching lines...) Expand all
316 int totalSteps = testCase.commands.length; 314 int totalSteps = testCase.commands.length;
317 String suffix =' (step $currentStep of $totalSteps)'; 315 String suffix =' (step $currentStep of $totalSteps)';
318 if (currentStep == totalSteps) { // done with test command 316 if (currentStep == totalSteps) { // done with test command
319 testComplete(exitCode); 317 testComplete(exitCode);
320 } else if (exitCode != 0) { 318 } else if (exitCode != 0) {
321 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); 319 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n');
322 testComplete(exitCode); 320 testComplete(exitCode);
323 } else { 321 } else {
324 stderr.add('test.dart: Compilion finished $suffix\n'); 322 stderr.add('test.dart: Compilion finished $suffix\n');
325 stdout.add('test.dart: Compilion finished $suffix\n'); 323 stdout.add('test.dart: Compilion finished $suffix\n');
326 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && 324 if (currentStep == totalSteps - 1
327 !testCase.configuration['noBatch']) { 325 && testCase.configuration['component'] == 'webdriver') {
328 // Note: processQueue will always be non-null for component == webdriver 326 // Note: processQueue will always be non-null for component == webdriver
329 // (It is only null for component == vm) 327 // (It is only null for component == vm)
330 processQueue._getBatchRunner(testCase).startTest(testCase); 328 processQueue._getBatchRunner(testCase).startTest(testCase);
331 } else { 329 } else {
332 runCommand(testCase.commands[currentStep++], stepExitHandler); 330 runCommand(testCase.commands[currentStep++], stepExitHandler);
333 } 331 }
334 } 332 }
335 } 333 }
336 334
337 Function makeReadHandler(StringInputStream source, List<String> destination) { 335 Function makeReadHandler(StringInputStream source, List<String> destination) {
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after
395 bool _stdoutDrained = false; 393 bool _stdoutDrained = false;
396 bool _stderrDrained = false; 394 bool _stderrDrained = false;
397 Date _startTime; 395 Date _startTime;
398 Timer _timer; 396 Timer _timer;
399 397
400 bool _isWebDriver; 398 bool _isWebDriver;
401 399
402 BatchRunnerProcess(TestCase testCase) { 400 BatchRunnerProcess(TestCase testCase) {
403 _executable = testCase.commands.last().executable; 401 _executable = testCase.commands.last().executable;
404 _batchArguments = testCase.batchRunnerArguments; 402 _batchArguments = testCase.batchRunnerArguments;
405 _isWebDriver = testCase.usesWebDriver; 403 _isWebDriver = testCase.configuration['component'] == 'webdriver';
406 } 404 }
407 405
408 bool get active() => _currentTest != null; 406 bool get active() => _currentTest != null;
409 407
410 void startTest(TestCase testCase) { 408 void startTest(TestCase testCase) {
411 _currentTest = testCase; 409 _currentTest = testCase;
412 if (_process === null) { 410 if (_process === null) {
413 // Start process if not yet started. 411 // Start process if not yet started.
414 _executable = testCase.commands.last().executable; 412 _executable = testCase.commands.last().executable;
415 _startProcess(() { 413 _startProcess(() {
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after
600 Map<String, List<BatchRunnerProcess>> _batchProcesses; 598 Map<String, List<BatchRunnerProcess>> _batchProcesses;
601 599
602 // Cache information about test cases per test suite. For multiple 600 // Cache information about test cases per test suite. For multiple
603 // configurations there is no need to repeatedly search the file 601 // configurations there is no need to repeatedly search the file
604 // system, generate tests, and search test files for options. 602 // system, generate tests, and search test files for options.
605 Map<String, List<TestInformation>> _testCache; 603 Map<String, List<TestInformation>> _testCache;
606 /** 604 /**
607 * String indicating the browser used to run the tests. Empty if no browser 605 * String indicating the browser used to run the tests. Empty if no browser
608 * used. 606 * used.
609 */ 607 */
610 String browserUsed = ''; 608 String browserUsed;
611 /**
612 * Process running the selenium server .jar (only used for Safari and Opera
613 * tests.)
614 */
615 Process _seleniumServer = null;
616 /** True if we are in the process of starting the server. */
617 bool _startingServer = false;
618 609
619 ProcessQueue(int this._maxProcesses, 610 ProcessQueue(int this._maxProcesses,
620 String progress, 611 String progress,
621 Date startTime, 612 Date startTime,
622 bool printTiming, 613 bool printTiming,
623 Function this._enqueueMoreWork, 614 Function this._enqueueMoreWork,
624 [bool this._verbose = false, 615 [bool this._verbose = false,
625 bool this._listTests = false, 616 bool this._listTests = false,
626 bool this._keepGeneratedTests = false]) 617 bool this._keepGeneratedTests = false])
627 : _tests = new Queue<TestCase>(), 618 : _tests = new Queue<TestCase>(),
628 _progress = new ProgressIndicator.fromName(progress, 619 _progress = new ProgressIndicator.fromName(progress,
629 startTime, 620 startTime,
630 printTiming), 621 printTiming),
631 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), 622 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(),
632 _testCache = new Map<String, List<TestInformation>>() { 623 _testCache = new Map<String, List<TestInformation>>() {
633 if (!_enqueueMoreWork(this)) _progress.allDone(); 624 if (!_enqueueMoreWork(this)) _progress.allDone();
625 browserUsed = '';
634 } 626 }
635 627
636 /** 628 /**
637 * Registers a TestSuite so that all of its tests will be run. 629 * Registers a TestSuite so that all of its tests will be run.
638 */ 630 */
639 void addTestSuite(TestSuite testSuite) { 631 void addTestSuite(TestSuite testSuite) {
640 _activeTestListers++; 632 _activeTestListers++;
641 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory, 633 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory,
642 _testListerDone); 634 _testListerDone);
643 } 635 }
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after
699 } 691 }
700 } 692 }
701 693
702 /** 694 /**
703 * Perform any cleanup needed once all tests in a TestSuite have completed 695 * Perform any cleanup needed once all tests in a TestSuite have completed
704 * and notify our progress indicator that we are done. 696 * and notify our progress indicator that we are done.
705 */ 697 */
706 void _cleanupAndMarkDone() { 698 void _cleanupAndMarkDone() {
707 if (browserUsed != '') { 699 if (browserUsed != '') {
708 killZombieBrowsers(); 700 killZombieBrowsers();
709 if (_isSeleniumAvailable) {
710 _seleniumServer.kill();
711 }
712 } else { 701 } else {
713 _progress.allDone(); 702 _progress.allDone();
714 } 703 }
715 } 704 }
716 705
717 void _checkDone() { 706 void _checkDone() {
718 // When there are no more active test listers ask for more work 707 // When there are no more active test listers ask for more work
719 // from process queue users. 708 // from process queue users.
720 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { 709 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) {
721 _progress.allTestsKnown(); 710 _progress.allTestsKnown();
(...skipping 18 matching lines...) Expand all
740 } 729 }
741 } else { 730 } else {
742 print('\nDeletion of temp dir $_temporaryDirectory failed.'); 731 print('\nDeletion of temp dir $_temporaryDirectory failed.');
743 } 732 }
744 _cleanupAndMarkDone(); 733 _cleanupAndMarkDone();
745 }; 734 };
746 } 735 }
747 } 736 }
748 } 737 }
749 } 738 }
750
751 /**
752 * True if we are using a browser + platform combination that needs the
753 * Selenium server jar.
754 */
755 bool get _needsSelenium() => new Platform().operatingSystem() == 'macos' &&
756 browserUsed == 'safari';
757
758 /** True if the Selenium Server is ready to be used. */
759 bool get _isSeleniumAvailable() => _seleniumServer != null;
760
761 /** Start the Selenium Server jar, if appropriate for this platform. */
762 void _ensureSeleniumServerRunning() {
763 if (!_isSeleniumAvailable && _startingServer == false) {
764 _startingServer = true;
765 _startSeleniumServer();
766 }
767 }
768 739
769 void _runTest(TestCase test) { 740 void _runTest(TestCase test) {
770 if (test.usesWebDriver) { 741 if (test.configuration['component'] == 'webdriver') {
771 browserUsed = test.configuration['browser']; 742 browserUsed = test.configuration['browser'];
772 if (_needsSelenium) _ensureSeleniumServerRunning();
773 } 743 }
774 _progress.testAdded(); 744 _progress.testAdded();
775 _tests.add(test); 745 _tests.add(test);
776 _tryRunTest(); 746 _tryRunTest();
777 } 747 }
778 748
779 /**
780 * Monitor the output of the Selenium server, to know when we are ready to
781 * begin running tests.
782 * source: Output(Stream) from the Java server.
783 */
784 Function makeSeleniumServerHandler(StringInputStream source) {
785 return () {
786 if (source.closed) return; // TODO(whesse): Remove when bug is fixed.
787 var line = source.readLine();
788 while (null != line) {
789 if (const RegExp(@".*Started.*Server.*").hasMatch(line) ||
790 const RegExp(@"Exception.*Selenium is already running.*").hasMatch(
791 line)) {
792 for (int i = 0; i < _maxProcesses; i++) {
793 // Restart all the processes that have been waiting/stopped for
794 // the server to start up. If we just call this once we end up
795 // with a single-"threaded" run.
796 _tryRunTest();
797 }
798 }
799 line = source.readLine();
800 }
801 };
802 }
803
804 /**
805 * For browser tests using Safari or Opera, we need to use the Selenium 1.0
806 * Java server.
807 */
808 void _startSeleniumServer() {
809 // Get the absolute path to the Selenium jar.
810 String filePath = new Options().script;
811 String pathSep = new Platform().pathSeparator();
812 int index = filePath.lastIndexOf(pathSep);
813 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}';
814 var dir = new Directory(filePath);
815 dir.onFile = (String file) {
816 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file)
817 && _seleniumServer == null) {
818 _seleniumServer = new Process.start('java', ['-jar', file]);
819 // Heads up: there seems to an obscure data race of some form in
820 // the VM between launching the server process and launching the test
821 // tasks that disappears when you read IO (which is convenient, since
822 // that is our condition for knowing that the server is ready).
823 StringInputStream stdoutStringStream =
824 new StringInputStream(_seleniumServer.stdout);
825 StringInputStream stderrStringStream =
826 new StringInputStream(_seleniumServer.stderr);
827 stdoutStringStream.onLine =
828 makeSeleniumServerHandler(stdoutStringStream);
829 stderrStringStream.onLine =
830 makeSeleniumServerHandler(stderrStringStream);
831 }
832 };
833 dir.list();
834 }
835
836 void _terminateBatchRunners() { 749 void _terminateBatchRunners() {
837 for (var runners in _batchProcesses.getValues()) { 750 for (var runners in _batchProcesses.getValues()) {
838 for (var runner in runners) { 751 for (var runner in runners) {
839 runner.terminate(); 752 runner.terminate();
840 } 753 }
841 } 754 }
842 } 755 }
843 756
844 BatchRunnerProcess _getBatchRunner(TestCase test) { 757 BatchRunnerProcess _getBatchRunner(TestCase test) {
845 // Start batch processes if needed 758 // Start batch processes if needed
(...skipping 19 matching lines...) Expand all
865 TestCase test = _tests.removeFirst(); 778 TestCase test = _tests.removeFirst();
866 if (_verbose) print(test.commands.last().commandLine); 779 if (_verbose) print(test.commands.last().commandLine);
867 if (_listTests) { 780 if (_listTests) {
868 final String tab = '\t'; 781 final String tab = '\t';
869 String outcomes = 782 String outcomes =
870 Strings.join(new List.from(test.expectedOutcomes), ','); 783 Strings.join(new List.from(test.expectedOutcomes), ',');
871 print(test.displayName + tab + outcomes + tab + test.isNegative + 784 print(test.displayName + tab + outcomes + tab + test.isNegative +
872 tab + Strings.join(test.commands.last().arguments, tab)); 785 tab + Strings.join(test.commands.last().arguments, tab));
873 return; 786 return;
874 } 787 }
875 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) {
876 // The server is not ready to run Selenium tests. Put them back in the
877 // queue.
878 _tests.addFirst(test);
879 return;
880 }
881 _progress.start(test); 788 _progress.start(test);
882 Function oldCallback = test.completedHandler; 789 Function oldCallback = test.completedHandler;
883 Function wrapper = (TestCase test_arg) { 790 Function wrapper = (TestCase test_arg) {
884 _numProcesses--; 791 _numProcesses--;
885 _progress.done(test_arg); 792 _progress.done(test_arg);
886 _tryRunTest(); 793 _tryRunTest();
887 oldCallback(test_arg); 794 oldCallback(test_arg);
888 }; 795 };
889 test.completedHandler = wrapper; 796 test.completedHandler = wrapper;
890 if (test.configuration['component'] == 'dartc' && 797 if (test.configuration['component'] == 'dartc' &&
891 test.displayName != 'dartc/junit_tests') { 798 test.displayName != 'dartc/junit_tests') {
892 _getBatchRunner(test).startTest(test); 799 _getBatchRunner(test).startTest(test);
893 } else { 800 } else {
894 // Once we've actually failed a test, technically, we wouldn't need to 801 // Once we've actually failed a test, technically, we wouldn't need to
895 // bother retrying any subsequent tests since the bot is already red. 802 // bother retrying any subsequent tests since the bot is already red.
896 // However, we continue to retry tests until we have actually failed 803 // However, we continue to retry tests until we have actually failed
897 // four tests (arbitrarily chosen) for more debugable output, so that 804 // four tests (arbitrarily chosen) for more debugable output, so that
898 // the developer doesn't waste his or her time trying to fix a bunch of 805 // the developer doesn't waste his or her time trying to fix a bunch of
899 // tests that appear to be broken but were actually just flakes that 806 // tests that appear to be broken but were actually just flakes that
900 // didn't get retried because there had already been one failure. 807 // didn't get retried because there had already been one failure.
901 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; 808 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
902 new RunningProcess(test, allowRetry, this).start(); 809 new RunningProcess(test, allowRetry, this).start();
903 } 810 }
904 _numProcesses++; 811 _numProcesses++;
905 } 812 }
906 } 813 }
907 } 814 }
OLDNEW
« no previous file with comments | « tools/testing/dart/test_options.dart ('k') | tools/testing/perf_testing/buildbot_browser_test_setup.sh » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698