OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
121 final component = configuration['component']; | 121 final component = configuration['component']; |
122 final mode = configuration['mode']; | 122 final mode = configuration['mode']; |
123 final arch = configuration['arch']; | 123 final arch = configuration['arch']; |
124 return "$component ${mode}_$arch"; | 124 return "$component ${mode}_$arch"; |
125 } | 125 } |
126 | 126 |
127 List<String> get batchRunnerArguments() => ['-batch']; | 127 List<String> get batchRunnerArguments() => ['-batch']; |
128 List<String> get batchTestArguments() => commands.last().arguments; | 128 List<String> get batchTestArguments() => commands.last().arguments; |
129 | 129 |
130 void completed() { completedHandler(this); } | 130 void completed() { completedHandler(this); } |
131 | |
132 bool get usesWebDriver() => configuration['component'] == 'webdriver'; | |
133 } | 131 } |
134 | 132 |
135 | 133 |
136 /** | 134 /** |
137 * BrowserTestCase has an extra compilation command that is run in a separate | 135 * BrowserTestCase has an extra compilation command that is run in a separate |
138 * process, before the regular test is run as in the base class [TestCase]. | 136 * process, before the regular test is run as in the base class [TestCase]. |
139 * If the compilation command fails, then the rest of the test is not run. | 137 * If the compilation command fails, then the rest of the test is not run. |
140 */ | 138 */ |
141 class BrowserTestCase extends TestCase { | 139 class BrowserTestCase extends TestCase { |
142 /** | 140 /** |
(...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
281 void testComplete(int exitCode) { | 279 void testComplete(int exitCode) { |
282 new TestOutput(testCase, exitCode, timedOut, stdout, | 280 new TestOutput(testCase, exitCode, timedOut, stdout, |
283 stderr, new Date.now().difference(startTime)); | 281 stderr, new Date.now().difference(startTime)); |
284 timeoutTimer.cancel(); | 282 timeoutTimer.cancel(); |
285 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { | 283 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { |
286 print(testCase.displayName); | 284 print(testCase.displayName); |
287 for (var line in testCase.output.stderr) print(line); | 285 for (var line in testCase.output.stderr) print(line); |
288 for (var line in testCase.output.stdout) print(line); | 286 for (var line in testCase.output.stdout) print(line); |
289 } | 287 } |
290 if (allowRetries != null && allowRetries | 288 if (allowRetries != null && allowRetries |
291 && testCase.usesWebDriver && testCase.output.unexpectedOutput | 289 && testCase.configuration['component'] == 'webdriver' && |
292 && testCase.numRetries > 0) { | 290 testCase.output.unexpectedOutput && testCase.numRetries > 0) { |
293 // Selenium tests can be flaky. Try rerunning. | 291 // Selenium tests can be flaky. Try rerunning. |
294 testCase.output.requestRetry = true; | 292 testCase.output.requestRetry = true; |
295 } | 293 } |
296 if (testCase.output.requestRetry) { | 294 if (testCase.output.requestRetry) { |
297 testCase.output.requestRetry = false; | 295 testCase.output.requestRetry = false; |
298 this.timedOut = false; | 296 this.timedOut = false; |
299 testCase.dynamic.numRetries--; | 297 testCase.dynamic.numRetries--; |
300 print("Potential flake. " + | 298 print("Potential flake. " + |
301 "Re-running ${testCase.displayName} " + | 299 "Re-running ${testCase.displayName} " + |
302 "(${testCase.dynamic.numRetries} attempt(s) remains)"); | 300 "(${testCase.dynamic.numRetries} attempt(s) remains)"); |
(...skipping 13 matching lines...) Expand all Loading... |
316 int totalSteps = testCase.commands.length; | 314 int totalSteps = testCase.commands.length; |
317 String suffix =' (step $currentStep of $totalSteps)'; | 315 String suffix =' (step $currentStep of $totalSteps)'; |
318 if (currentStep == totalSteps) { // done with test command | 316 if (currentStep == totalSteps) { // done with test command |
319 testComplete(exitCode); | 317 testComplete(exitCode); |
320 } else if (exitCode != 0) { | 318 } else if (exitCode != 0) { |
321 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 319 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
322 testComplete(exitCode); | 320 testComplete(exitCode); |
323 } else { | 321 } else { |
324 stderr.add('test.dart: Compilion finished $suffix\n'); | 322 stderr.add('test.dart: Compilion finished $suffix\n'); |
325 stdout.add('test.dart: Compilion finished $suffix\n'); | 323 stdout.add('test.dart: Compilion finished $suffix\n'); |
326 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && | 324 if (currentStep == totalSteps - 1 |
327 !testCase.configuration['noBatch']) { | 325 && testCase.configuration['component'] == 'webdriver') { |
328 // Note: processQueue will always be non-null for component == webdriver | 326 // Note: processQueue will always be non-null for component == webdriver |
329 // (It is only null for component == vm) | 327 // (It is only null for component == vm) |
330 processQueue._getBatchRunner(testCase).startTest(testCase); | 328 processQueue._getBatchRunner(testCase).startTest(testCase); |
331 } else { | 329 } else { |
332 runCommand(testCase.commands[currentStep++], stepExitHandler); | 330 runCommand(testCase.commands[currentStep++], stepExitHandler); |
333 } | 331 } |
334 } | 332 } |
335 } | 333 } |
336 | 334 |
337 Function makeReadHandler(StringInputStream source, List<String> destination) { | 335 Function makeReadHandler(StringInputStream source, List<String> destination) { |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
395 bool _stdoutDrained = false; | 393 bool _stdoutDrained = false; |
396 bool _stderrDrained = false; | 394 bool _stderrDrained = false; |
397 Date _startTime; | 395 Date _startTime; |
398 Timer _timer; | 396 Timer _timer; |
399 | 397 |
400 bool _isWebDriver; | 398 bool _isWebDriver; |
401 | 399 |
402 BatchRunnerProcess(TestCase testCase) { | 400 BatchRunnerProcess(TestCase testCase) { |
403 _executable = testCase.commands.last().executable; | 401 _executable = testCase.commands.last().executable; |
404 _batchArguments = testCase.batchRunnerArguments; | 402 _batchArguments = testCase.batchRunnerArguments; |
405 _isWebDriver = testCase.usesWebDriver; | 403 _isWebDriver = testCase.configuration['component'] == 'webdriver'; |
406 } | 404 } |
407 | 405 |
408 bool get active() => _currentTest != null; | 406 bool get active() => _currentTest != null; |
409 | 407 |
410 void startTest(TestCase testCase) { | 408 void startTest(TestCase testCase) { |
411 _currentTest = testCase; | 409 _currentTest = testCase; |
412 if (_process === null) { | 410 if (_process === null) { |
413 // Start process if not yet started. | 411 // Start process if not yet started. |
414 _executable = testCase.commands.last().executable; | 412 _executable = testCase.commands.last().executable; |
415 _startProcess(() { | 413 _startProcess(() { |
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
600 Map<String, List<BatchRunnerProcess>> _batchProcesses; | 598 Map<String, List<BatchRunnerProcess>> _batchProcesses; |
601 | 599 |
602 // Cache information about test cases per test suite. For multiple | 600 // Cache information about test cases per test suite. For multiple |
603 // configurations there is no need to repeatedly search the file | 601 // configurations there is no need to repeatedly search the file |
604 // system, generate tests, and search test files for options. | 602 // system, generate tests, and search test files for options. |
605 Map<String, List<TestInformation>> _testCache; | 603 Map<String, List<TestInformation>> _testCache; |
606 /** | 604 /** |
607 * String indicating the browser used to run the tests. Empty if no browser | 605 * String indicating the browser used to run the tests. Empty if no browser |
608 * used. | 606 * used. |
609 */ | 607 */ |
610 String browserUsed = ''; | 608 String browserUsed; |
611 /** | |
612 * Process running the selenium server .jar (only used for Safari and Opera | |
613 * tests.) | |
614 */ | |
615 Process _seleniumServer = null; | |
616 /** True if we are in the process of starting the server. */ | |
617 bool _startingServer = false; | |
618 | 609 |
619 ProcessQueue(int this._maxProcesses, | 610 ProcessQueue(int this._maxProcesses, |
620 String progress, | 611 String progress, |
621 Date startTime, | 612 Date startTime, |
622 bool printTiming, | 613 bool printTiming, |
623 Function this._enqueueMoreWork, | 614 Function this._enqueueMoreWork, |
624 [bool this._verbose = false, | 615 [bool this._verbose = false, |
625 bool this._listTests = false, | 616 bool this._listTests = false, |
626 bool this._keepGeneratedTests = false]) | 617 bool this._keepGeneratedTests = false]) |
627 : _tests = new Queue<TestCase>(), | 618 : _tests = new Queue<TestCase>(), |
628 _progress = new ProgressIndicator.fromName(progress, | 619 _progress = new ProgressIndicator.fromName(progress, |
629 startTime, | 620 startTime, |
630 printTiming), | 621 printTiming), |
631 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), | 622 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), |
632 _testCache = new Map<String, List<TestInformation>>() { | 623 _testCache = new Map<String, List<TestInformation>>() { |
633 if (!_enqueueMoreWork(this)) _progress.allDone(); | 624 if (!_enqueueMoreWork(this)) _progress.allDone(); |
| 625 browserUsed = ''; |
634 } | 626 } |
635 | 627 |
636 /** | 628 /** |
637 * Registers a TestSuite so that all of its tests will be run. | 629 * Registers a TestSuite so that all of its tests will be run. |
638 */ | 630 */ |
639 void addTestSuite(TestSuite testSuite) { | 631 void addTestSuite(TestSuite testSuite) { |
640 _activeTestListers++; | 632 _activeTestListers++; |
641 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory, | 633 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory, |
642 _testListerDone); | 634 _testListerDone); |
643 } | 635 } |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
699 } | 691 } |
700 } | 692 } |
701 | 693 |
702 /** | 694 /** |
703 * Perform any cleanup needed once all tests in a TestSuite have completed | 695 * Perform any cleanup needed once all tests in a TestSuite have completed |
704 * and notify our progress indicator that we are done. | 696 * and notify our progress indicator that we are done. |
705 */ | 697 */ |
706 void _cleanupAndMarkDone() { | 698 void _cleanupAndMarkDone() { |
707 if (browserUsed != '') { | 699 if (browserUsed != '') { |
708 killZombieBrowsers(); | 700 killZombieBrowsers(); |
709 if (_isSeleniumAvailable) { | |
710 _seleniumServer.kill(); | |
711 } | |
712 } else { | 701 } else { |
713 _progress.allDone(); | 702 _progress.allDone(); |
714 } | 703 } |
715 } | 704 } |
716 | 705 |
717 void _checkDone() { | 706 void _checkDone() { |
718 // When there are no more active test listers ask for more work | 707 // When there are no more active test listers ask for more work |
719 // from process queue users. | 708 // from process queue users. |
720 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { | 709 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { |
721 _progress.allTestsKnown(); | 710 _progress.allTestsKnown(); |
(...skipping 18 matching lines...) Expand all Loading... |
740 } | 729 } |
741 } else { | 730 } else { |
742 print('\nDeletion of temp dir $_temporaryDirectory failed.'); | 731 print('\nDeletion of temp dir $_temporaryDirectory failed.'); |
743 } | 732 } |
744 _cleanupAndMarkDone(); | 733 _cleanupAndMarkDone(); |
745 }; | 734 }; |
746 } | 735 } |
747 } | 736 } |
748 } | 737 } |
749 } | 738 } |
750 | |
751 /** | |
752 * True if we are using a browser + platform combination that needs the | |
753 * Selenium server jar. | |
754 */ | |
755 bool get _needsSelenium() => new Platform().operatingSystem() == 'macos' && | |
756 browserUsed == 'safari'; | |
757 | |
758 /** True if the Selenium Server is ready to be used. */ | |
759 bool get _isSeleniumAvailable() => _seleniumServer != null; | |
760 | |
761 /** Start the Selenium Server jar, if appropriate for this platform. */ | |
762 void _ensureSeleniumServerRunning() { | |
763 if (!_isSeleniumAvailable && _startingServer == false) { | |
764 _startingServer = true; | |
765 _startSeleniumServer(); | |
766 } | |
767 } | |
768 | 739 |
769 void _runTest(TestCase test) { | 740 void _runTest(TestCase test) { |
770 if (test.usesWebDriver) { | 741 if (test.configuration['component'] == 'webdriver') { |
771 browserUsed = test.configuration['browser']; | 742 browserUsed = test.configuration['browser']; |
772 if (_needsSelenium) _ensureSeleniumServerRunning(); | |
773 } | 743 } |
774 _progress.testAdded(); | 744 _progress.testAdded(); |
775 _tests.add(test); | 745 _tests.add(test); |
776 _tryRunTest(); | 746 _tryRunTest(); |
777 } | 747 } |
778 | 748 |
779 /** | |
780 * Monitor the output of the Selenium server, to know when we are ready to | |
781 * begin running tests. | |
782 * source: Output(Stream) from the Java server. | |
783 */ | |
784 Function makeSeleniumServerHandler(StringInputStream source) { | |
785 return () { | |
786 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. | |
787 var line = source.readLine(); | |
788 while (null != line) { | |
789 if (const RegExp(@".*Started.*Server.*").hasMatch(line) || | |
790 const RegExp(@"Exception.*Selenium is already running.*").hasMatch( | |
791 line)) { | |
792 for (int i = 0; i < _maxProcesses; i++) { | |
793 // Restart all the processes that have been waiting/stopped for | |
794 // the server to start up. If we just call this once we end up | |
795 // with a single-"threaded" run. | |
796 _tryRunTest(); | |
797 } | |
798 } | |
799 line = source.readLine(); | |
800 } | |
801 }; | |
802 } | |
803 | |
804 /** | |
805 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 | |
806 * Java server. | |
807 */ | |
808 void _startSeleniumServer() { | |
809 // Get the absolute path to the Selenium jar. | |
810 String filePath = new Options().script; | |
811 String pathSep = new Platform().pathSeparator(); | |
812 int index = filePath.lastIndexOf(pathSep); | |
813 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; | |
814 var dir = new Directory(filePath); | |
815 dir.onFile = (String file) { | |
816 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) | |
817 && _seleniumServer == null) { | |
818 _seleniumServer = new Process.start('java', ['-jar', file]); | |
819 // Heads up: there seems to an obscure data race of some form in | |
820 // the VM between launching the server process and launching the test | |
821 // tasks that disappears when you read IO (which is convenient, since | |
822 // that is our condition for knowing that the server is ready). | |
823 StringInputStream stdoutStringStream = | |
824 new StringInputStream(_seleniumServer.stdout); | |
825 StringInputStream stderrStringStream = | |
826 new StringInputStream(_seleniumServer.stderr); | |
827 stdoutStringStream.onLine = | |
828 makeSeleniumServerHandler(stdoutStringStream); | |
829 stderrStringStream.onLine = | |
830 makeSeleniumServerHandler(stderrStringStream); | |
831 } | |
832 }; | |
833 dir.list(); | |
834 } | |
835 | |
836 void _terminateBatchRunners() { | 749 void _terminateBatchRunners() { |
837 for (var runners in _batchProcesses.getValues()) { | 750 for (var runners in _batchProcesses.getValues()) { |
838 for (var runner in runners) { | 751 for (var runner in runners) { |
839 runner.terminate(); | 752 runner.terminate(); |
840 } | 753 } |
841 } | 754 } |
842 } | 755 } |
843 | 756 |
844 BatchRunnerProcess _getBatchRunner(TestCase test) { | 757 BatchRunnerProcess _getBatchRunner(TestCase test) { |
845 // Start batch processes if needed | 758 // Start batch processes if needed |
(...skipping 19 matching lines...) Expand all Loading... |
865 TestCase test = _tests.removeFirst(); | 778 TestCase test = _tests.removeFirst(); |
866 if (_verbose) print(test.commands.last().commandLine); | 779 if (_verbose) print(test.commands.last().commandLine); |
867 if (_listTests) { | 780 if (_listTests) { |
868 final String tab = '\t'; | 781 final String tab = '\t'; |
869 String outcomes = | 782 String outcomes = |
870 Strings.join(new List.from(test.expectedOutcomes), ','); | 783 Strings.join(new List.from(test.expectedOutcomes), ','); |
871 print(test.displayName + tab + outcomes + tab + test.isNegative + | 784 print(test.displayName + tab + outcomes + tab + test.isNegative + |
872 tab + Strings.join(test.commands.last().arguments, tab)); | 785 tab + Strings.join(test.commands.last().arguments, tab)); |
873 return; | 786 return; |
874 } | 787 } |
875 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { | |
876 // The server is not ready to run Selenium tests. Put them back in the | |
877 // queue. | |
878 _tests.addFirst(test); | |
879 return; | |
880 } | |
881 _progress.start(test); | 788 _progress.start(test); |
882 Function oldCallback = test.completedHandler; | 789 Function oldCallback = test.completedHandler; |
883 Function wrapper = (TestCase test_arg) { | 790 Function wrapper = (TestCase test_arg) { |
884 _numProcesses--; | 791 _numProcesses--; |
885 _progress.done(test_arg); | 792 _progress.done(test_arg); |
886 _tryRunTest(); | 793 _tryRunTest(); |
887 oldCallback(test_arg); | 794 oldCallback(test_arg); |
888 }; | 795 }; |
889 test.completedHandler = wrapper; | 796 test.completedHandler = wrapper; |
890 if (test.configuration['component'] == 'dartc' && | 797 if (test.configuration['component'] == 'dartc' && |
891 test.displayName != 'dartc/junit_tests') { | 798 test.displayName != 'dartc/junit_tests') { |
892 _getBatchRunner(test).startTest(test); | 799 _getBatchRunner(test).startTest(test); |
893 } else { | 800 } else { |
894 // Once we've actually failed a test, technically, we wouldn't need to | 801 // Once we've actually failed a test, technically, we wouldn't need to |
895 // bother retrying any subsequent tests since the bot is already red. | 802 // bother retrying any subsequent tests since the bot is already red. |
896 // However, we continue to retry tests until we have actually failed | 803 // However, we continue to retry tests until we have actually failed |
897 // four tests (arbitrarily chosen) for more debugable output, so that | 804 // four tests (arbitrarily chosen) for more debugable output, so that |
898 // the developer doesn't waste his or her time trying to fix a bunch of | 805 // the developer doesn't waste his or her time trying to fix a bunch of |
899 // tests that appear to be broken but were actually just flakes that | 806 // tests that appear to be broken but were actually just flakes that |
900 // didn't get retried because there had already been one failure. | 807 // didn't get retried because there had already been one failure. |
901 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 808 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
902 new RunningProcess(test, allowRetry, this).start(); | 809 new RunningProcess(test, allowRetry, this).start(); |
903 } | 810 } |
904 _numProcesses++; | 811 _numProcesses++; |
905 } | 812 } |
906 } | 813 } |
907 } | 814 } |
OLD | NEW |