OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
121 final component = configuration['component']; | 121 final component = configuration['component']; |
122 final mode = configuration['mode']; | 122 final mode = configuration['mode']; |
123 final arch = configuration['arch']; | 123 final arch = configuration['arch']; |
124 return "$component ${mode}_$arch"; | 124 return "$component ${mode}_$arch"; |
125 } | 125 } |
126 | 126 |
127 List<String> get batchRunnerArguments() => ['-batch']; | 127 List<String> get batchRunnerArguments() => ['-batch']; |
128 List<String> get batchTestArguments() => commands.last().arguments; | 128 List<String> get batchTestArguments() => commands.last().arguments; |
129 | 129 |
130 void completed() { completedHandler(this); } | 130 void completed() { completedHandler(this); } |
| 131 |
| 132 bool get usesWebDriver() => configuration['component'] == 'webdriver'; |
131 } | 133 } |
132 | 134 |
133 | 135 |
134 /** | 136 /** |
135 * BrowserTestCase has an extra compilation command that is run in a separate | 137 * BrowserTestCase has an extra compilation command that is run in a separate |
136 * process, before the regular test is run as in the base class [TestCase]. | 138 * process, before the regular test is run as in the base class [TestCase]. |
137 * If the compilation command fails, then the rest of the test is not run. | 139 * If the compilation command fails, then the rest of the test is not run. |
138 */ | 140 */ |
139 class BrowserTestCase extends TestCase { | 141 class BrowserTestCase extends TestCase { |
140 /** | 142 /** |
(...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
279 void testComplete(int exitCode) { | 281 void testComplete(int exitCode) { |
280 new TestOutput(testCase, exitCode, timedOut, stdout, | 282 new TestOutput(testCase, exitCode, timedOut, stdout, |
281 stderr, new Date.now().difference(startTime)); | 283 stderr, new Date.now().difference(startTime)); |
282 timeoutTimer.cancel(); | 284 timeoutTimer.cancel(); |
283 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { | 285 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { |
284 print(testCase.displayName); | 286 print(testCase.displayName); |
285 for (var line in testCase.output.stderr) print(line); | 287 for (var line in testCase.output.stderr) print(line); |
286 for (var line in testCase.output.stdout) print(line); | 288 for (var line in testCase.output.stdout) print(line); |
287 } | 289 } |
288 if (allowRetries != null && allowRetries | 290 if (allowRetries != null && allowRetries |
289 && testCase.configuration['component'] == 'webdriver' && | 291 && testCase.usesWebDriver && testCase.output.unexpectedOutput |
290 testCase.output.unexpectedOutput && testCase.numRetries > 0) { | 292 && testCase.numRetries > 0) { |
291 // Selenium tests can be flaky. Try rerunning. | 293 // Selenium tests can be flaky. Try rerunning. |
292 testCase.output.requestRetry = true; | 294 testCase.output.requestRetry = true; |
293 } | 295 } |
294 if (testCase.output.requestRetry) { | 296 if (testCase.output.requestRetry) { |
295 testCase.output.requestRetry = false; | 297 testCase.output.requestRetry = false; |
296 this.timedOut = false; | 298 this.timedOut = false; |
297 testCase.dynamic.numRetries--; | 299 testCase.dynamic.numRetries--; |
298 print("Potential flake. " + | 300 print("Potential flake. " + |
299 "Re-running ${testCase.displayName} " + | 301 "Re-running ${testCase.displayName} " + |
300 "(${testCase.dynamic.numRetries} attempt(s) remains)"); | 302 "(${testCase.dynamic.numRetries} attempt(s) remains)"); |
(...skipping 13 matching lines...) Expand all Loading... |
314 int totalSteps = testCase.commands.length; | 316 int totalSteps = testCase.commands.length; |
315 String suffix =' (step $currentStep of $totalSteps)'; | 317 String suffix =' (step $currentStep of $totalSteps)'; |
316 if (currentStep == totalSteps) { // done with test command | 318 if (currentStep == totalSteps) { // done with test command |
317 testComplete(exitCode); | 319 testComplete(exitCode); |
318 } else if (exitCode != 0) { | 320 } else if (exitCode != 0) { |
319 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 321 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
320 testComplete(exitCode); | 322 testComplete(exitCode); |
321 } else { | 323 } else { |
322 stderr.add('test.dart: Compilion finished $suffix\n'); | 324 stderr.add('test.dart: Compilion finished $suffix\n'); |
323 stdout.add('test.dart: Compilion finished $suffix\n'); | 325 stdout.add('test.dart: Compilion finished $suffix\n'); |
324 if (currentStep == totalSteps - 1 | 326 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && |
325 && testCase.configuration['component'] == 'webdriver') { | 327 !testCase.configuration['noBatch']) { |
326 // Note: processQueue will always be non-null for component == webdriver | 328 // Note: processQueue will always be non-null for component == webdriver |
327 // (It is only null for component == vm) | 329 // (It is only null for component == vm) |
328 processQueue._getBatchRunner(testCase).startTest(testCase); | 330 processQueue._getBatchRunner(testCase).startTest(testCase); |
329 } else { | 331 } else { |
330 runCommand(testCase.commands[currentStep++], stepExitHandler); | 332 runCommand(testCase.commands[currentStep++], stepExitHandler); |
331 } | 333 } |
332 } | 334 } |
333 } | 335 } |
334 | 336 |
335 Function makeReadHandler(StringInputStream source, List<String> destination) { | 337 Function makeReadHandler(StringInputStream source, List<String> destination) { |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
393 bool _stdoutDrained = false; | 395 bool _stdoutDrained = false; |
394 bool _stderrDrained = false; | 396 bool _stderrDrained = false; |
395 Date _startTime; | 397 Date _startTime; |
396 Timer _timer; | 398 Timer _timer; |
397 | 399 |
398 bool _isWebDriver; | 400 bool _isWebDriver; |
399 | 401 |
400 BatchRunnerProcess(TestCase testCase) { | 402 BatchRunnerProcess(TestCase testCase) { |
401 _executable = testCase.commands.last().executable; | 403 _executable = testCase.commands.last().executable; |
402 _batchArguments = testCase.batchRunnerArguments; | 404 _batchArguments = testCase.batchRunnerArguments; |
403 _isWebDriver = testCase.configuration['component'] == 'webdriver'; | 405 _isWebDriver = testCase.usesWebDriver; |
404 } | 406 } |
405 | 407 |
406 bool get active() => _currentTest != null; | 408 bool get active() => _currentTest != null; |
407 | 409 |
408 void startTest(TestCase testCase) { | 410 void startTest(TestCase testCase) { |
409 _currentTest = testCase; | 411 _currentTest = testCase; |
410 if (_process === null) { | 412 if (_process === null) { |
411 // Start process if not yet started. | 413 // Start process if not yet started. |
412 _executable = testCase.commands.last().executable; | 414 _executable = testCase.commands.last().executable; |
413 _startProcess(() { | 415 _startProcess(() { |
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
598 Map<String, List<BatchRunnerProcess>> _batchProcesses; | 600 Map<String, List<BatchRunnerProcess>> _batchProcesses; |
599 | 601 |
600 // Cache information about test cases per test suite. For multiple | 602 // Cache information about test cases per test suite. For multiple |
601 // configurations there is no need to repeatedly search the file | 603 // configurations there is no need to repeatedly search the file |
602 // system, generate tests, and search test files for options. | 604 // system, generate tests, and search test files for options. |
603 Map<String, List<TestInformation>> _testCache; | 605 Map<String, List<TestInformation>> _testCache; |
604 /** | 606 /** |
605 * String indicating the browser used to run the tests. Empty if no browser | 607 * String indicating the browser used to run the tests. Empty if no browser |
606 * used. | 608 * used. |
607 */ | 609 */ |
608 String browserUsed; | 610 String browserUsed = ''; |
| 611 /** |
| 612 * Process running the selenium server .jar (only used for Safari and Opera |
| 613 * tests.) |
| 614 */ |
| 615 Process _seleniumServer = null; |
| 616 /** True if we are in the process of starting the server. */ |
| 617 bool _startingServer = false; |
609 | 618 |
610 ProcessQueue(int this._maxProcesses, | 619 ProcessQueue(int this._maxProcesses, |
611 String progress, | 620 String progress, |
612 Date startTime, | 621 Date startTime, |
613 bool printTiming, | 622 bool printTiming, |
614 Function this._enqueueMoreWork, | 623 Function this._enqueueMoreWork, |
615 [bool this._verbose = false, | 624 [bool this._verbose = false, |
616 bool this._listTests = false, | 625 bool this._listTests = false, |
617 bool this._keepGeneratedTests = false]) | 626 bool this._keepGeneratedTests = false]) |
618 : _tests = new Queue<TestCase>(), | 627 : _tests = new Queue<TestCase>(), |
619 _progress = new ProgressIndicator.fromName(progress, | 628 _progress = new ProgressIndicator.fromName(progress, |
620 startTime, | 629 startTime, |
621 printTiming), | 630 printTiming), |
622 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), | 631 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), |
623 _testCache = new Map<String, List<TestInformation>>() { | 632 _testCache = new Map<String, List<TestInformation>>() { |
624 if (!_enqueueMoreWork(this)) _progress.allDone(); | 633 if (!_enqueueMoreWork(this)) _progress.allDone(); |
625 browserUsed = ''; | |
626 } | 634 } |
627 | 635 |
628 /** | 636 /** |
629 * Registers a TestSuite so that all of its tests will be run. | 637 * Registers a TestSuite so that all of its tests will be run. |
630 */ | 638 */ |
631 void addTestSuite(TestSuite testSuite) { | 639 void addTestSuite(TestSuite testSuite) { |
632 _activeTestListers++; | 640 _activeTestListers++; |
633 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory, | 641 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory, |
634 _testListerDone); | 642 _testListerDone); |
635 } | 643 } |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
691 } | 699 } |
692 } | 700 } |
693 | 701 |
694 /** | 702 /** |
695 * Perform any cleanup needed once all tests in a TestSuite have completed | 703 * Perform any cleanup needed once all tests in a TestSuite have completed |
696 * and notify our progress indicator that we are done. | 704 * and notify our progress indicator that we are done. |
697 */ | 705 */ |
698 void _cleanupAndMarkDone() { | 706 void _cleanupAndMarkDone() { |
699 if (browserUsed != '') { | 707 if (browserUsed != '') { |
700 killZombieBrowsers(); | 708 killZombieBrowsers(); |
| 709 if (_isSeleniumAvailable) { |
| 710 _seleniumServer.kill(); |
| 711 } |
701 } else { | 712 } else { |
702 _progress.allDone(); | 713 _progress.allDone(); |
703 } | 714 } |
704 } | 715 } |
705 | 716 |
706 void _checkDone() { | 717 void _checkDone() { |
707 // When there are no more active test listers ask for more work | 718 // When there are no more active test listers ask for more work |
708 // from process queue users. | 719 // from process queue users. |
709 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { | 720 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { |
710 _progress.allTestsKnown(); | 721 _progress.allTestsKnown(); |
(...skipping 18 matching lines...) Expand all Loading... |
729 } | 740 } |
730 } else { | 741 } else { |
731 print('\nDeletion of temp dir $_temporaryDirectory failed.'); | 742 print('\nDeletion of temp dir $_temporaryDirectory failed.'); |
732 } | 743 } |
733 _cleanupAndMarkDone(); | 744 _cleanupAndMarkDone(); |
734 }; | 745 }; |
735 } | 746 } |
736 } | 747 } |
737 } | 748 } |
738 } | 749 } |
| 750 |
| 751 /** |
| 752 * True if we are using a browser + platform combination that needs the |
| 753 * Selenium server jar. |
| 754 */ |
| 755 bool get _needsSelenium() => new Platform().operatingSystem() == 'macos' && |
| 756 browserUsed == 'safari'; |
| 757 |
| 758 /** True if the Selenium Server is ready to be used. */ |
| 759 bool get _isSeleniumAvailable() => _seleniumServer != null; |
| 760 |
| 761 /** Start the Selenium Server jar, if appropriate for this platform. */ |
| 762 void _ensureSeleniumServerRunning() { |
| 763 if (!_isSeleniumAvailable && _startingServer == false) { |
| 764 _startingServer = true; |
| 765 _startSeleniumServer(); |
| 766 } |
| 767 } |
739 | 768 |
740 void _runTest(TestCase test) { | 769 void _runTest(TestCase test) { |
741 if (test.configuration['component'] == 'webdriver') { | 770 if (test.usesWebDriver) { |
742 browserUsed = test.configuration['browser']; | 771 browserUsed = test.configuration['browser']; |
| 772 if (_needsSelenium) _ensureSeleniumServerRunning(); |
743 } | 773 } |
744 _progress.testAdded(); | 774 _progress.testAdded(); |
745 _tests.add(test); | 775 _tests.add(test); |
746 _tryRunTest(); | 776 _tryRunTest(); |
747 } | 777 } |
748 | 778 |
| 779 /** |
| 780 * Monitor the output of the Selenium server, to know when we are ready to |
| 781 * begin running tests. |
| 782 * source: Output(Stream) from the Java server. |
| 783 */ |
| 784 Function makeSeleniumServerHandler(StringInputStream source) { |
| 785 return () { |
| 786 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. |
| 787 var line = source.readLine(); |
| 788 while (null != line) { |
| 789 if (const RegExp(@".*Started.*Server.*").hasMatch(line) || |
| 790 const RegExp(@"Exception.*Selenium is already running.*").hasMatch( |
| 791 line)) { |
| 792 for (int i = 0; i < _maxProcesses; i++) { |
| 793 // Restart all the processes that have been waiting/stopped for |
| 794 // the server to start up. If we just call this once we end up |
| 795 // with a single-"threaded" run. |
| 796 _tryRunTest(); |
| 797 } |
| 798 } |
| 799 line = source.readLine(); |
| 800 } |
| 801 }; |
| 802 } |
| 803 |
| 804 /** |
| 805 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 |
| 806 * Java server. |
| 807 */ |
| 808 void _startSeleniumServer() { |
| 809 // Get the absolute path to the Selenium jar. |
| 810 String filePath = new Options().script; |
| 811 String pathSep = new Platform().pathSeparator(); |
| 812 int index = filePath.lastIndexOf(pathSep); |
| 813 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; |
| 814 var dir = new Directory(filePath); |
| 815 dir.onFile = (String file) { |
| 816 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) |
| 817 && _seleniumServer == null) { |
| 818 _seleniumServer = new Process.start('java', ['-jar', file]); |
| 819 // Heads up: there seems to an obscure data race of some form in |
| 820 // the VM between launching the server process and launching the test |
| 821 // tasks that disappears when you read IO (which is convenient, since |
| 822 // that is our condition for knowing that the server is ready). |
| 823 StringInputStream stdoutStringStream = |
| 824 new StringInputStream(_seleniumServer.stdout); |
| 825 StringInputStream stderrStringStream = |
| 826 new StringInputStream(_seleniumServer.stderr); |
| 827 stdoutStringStream.onLine = |
| 828 makeSeleniumServerHandler(stdoutStringStream); |
| 829 stderrStringStream.onLine = |
| 830 makeSeleniumServerHandler(stderrStringStream); |
| 831 } |
| 832 }; |
| 833 dir.list(); |
| 834 } |
| 835 |
749 void _terminateBatchRunners() { | 836 void _terminateBatchRunners() { |
750 for (var runners in _batchProcesses.getValues()) { | 837 for (var runners in _batchProcesses.getValues()) { |
751 for (var runner in runners) { | 838 for (var runner in runners) { |
752 runner.terminate(); | 839 runner.terminate(); |
753 } | 840 } |
754 } | 841 } |
755 } | 842 } |
756 | 843 |
757 BatchRunnerProcess _getBatchRunner(TestCase test) { | 844 BatchRunnerProcess _getBatchRunner(TestCase test) { |
758 // Start batch processes if needed | 845 // Start batch processes if needed |
(...skipping 19 matching lines...) Expand all Loading... |
778 TestCase test = _tests.removeFirst(); | 865 TestCase test = _tests.removeFirst(); |
779 if (_verbose) print(test.commands.last().commandLine); | 866 if (_verbose) print(test.commands.last().commandLine); |
780 if (_listTests) { | 867 if (_listTests) { |
781 final String tab = '\t'; | 868 final String tab = '\t'; |
782 String outcomes = | 869 String outcomes = |
783 Strings.join(new List.from(test.expectedOutcomes), ','); | 870 Strings.join(new List.from(test.expectedOutcomes), ','); |
784 print(test.displayName + tab + outcomes + tab + test.isNegative + | 871 print(test.displayName + tab + outcomes + tab + test.isNegative + |
785 tab + Strings.join(test.commands.last().arguments, tab)); | 872 tab + Strings.join(test.commands.last().arguments, tab)); |
786 return; | 873 return; |
787 } | 874 } |
| 875 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { |
| 876 // The server is not ready to run Selenium tests. Put them back in the |
| 877 // queue. |
| 878 _tests.addFirst(test); |
| 879 return; |
| 880 } |
788 _progress.start(test); | 881 _progress.start(test); |
789 Function oldCallback = test.completedHandler; | 882 Function oldCallback = test.completedHandler; |
790 Function wrapper = (TestCase test_arg) { | 883 Function wrapper = (TestCase test_arg) { |
791 _numProcesses--; | 884 _numProcesses--; |
792 _progress.done(test_arg); | 885 _progress.done(test_arg); |
793 _tryRunTest(); | 886 _tryRunTest(); |
794 oldCallback(test_arg); | 887 oldCallback(test_arg); |
795 }; | 888 }; |
796 test.completedHandler = wrapper; | 889 test.completedHandler = wrapper; |
797 if (test.configuration['component'] == 'dartc' && | 890 if (test.configuration['component'] == 'dartc' && |
798 test.displayName != 'dartc/junit_tests') { | 891 test.displayName != 'dartc/junit_tests') { |
799 _getBatchRunner(test).startTest(test); | 892 _getBatchRunner(test).startTest(test); |
800 } else { | 893 } else { |
801 // Once we've actually failed a test, technically, we wouldn't need to | 894 // Once we've actually failed a test, technically, we wouldn't need to |
802 // bother retrying any subsequent tests since the bot is already red. | 895 // bother retrying any subsequent tests since the bot is already red. |
803 // However, we continue to retry tests until we have actually failed | 896 // However, we continue to retry tests until we have actually failed |
804 // four tests (arbitrarily chosen) for more debugable output, so that | 897 // four tests (arbitrarily chosen) for more debugable output, so that |
805 // the developer doesn't waste his or her time trying to fix a bunch of | 898 // the developer doesn't waste his or her time trying to fix a bunch of |
806 // tests that appear to be broken but were actually just flakes that | 899 // tests that appear to be broken but were actually just flakes that |
807 // didn't get retried because there had already been one failure. | 900 // didn't get retried because there had already been one failure. |
808 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 901 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
809 new RunningProcess(test, allowRetry, this).start(); | 902 new RunningProcess(test, allowRetry, this).start(); |
810 } | 903 } |
811 _numProcesses++; | 904 _numProcesses++; |
812 } | 905 } |
813 } | 906 } |
814 } | 907 } |
OLD | NEW |