| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 121 final component = configuration['component']; | 121 final component = configuration['component']; |
| 122 final mode = configuration['mode']; | 122 final mode = configuration['mode']; |
| 123 final arch = configuration['arch']; | 123 final arch = configuration['arch']; |
| 124 return "$component ${mode}_$arch"; | 124 return "$component ${mode}_$arch"; |
| 125 } | 125 } |
| 126 | 126 |
| 127 List<String> get batchRunnerArguments() => ['-batch']; | 127 List<String> get batchRunnerArguments() => ['-batch']; |
| 128 List<String> get batchTestArguments() => commands.last().arguments; | 128 List<String> get batchTestArguments() => commands.last().arguments; |
| 129 | 129 |
| 130 void completed() { completedHandler(this); } | 130 void completed() { completedHandler(this); } |
| 131 | |
| 132 bool get usesWebDriver() => configuration['component'] == 'webdriver'; | |
| 133 } | 131 } |
| 134 | 132 |
| 135 | 133 |
| 136 /** | 134 /** |
| 137 * BrowserTestCase has an extra compilation command that is run in a separate | 135 * BrowserTestCase has an extra compilation command that is run in a separate |
| 138 * process, before the regular test is run as in the base class [TestCase]. | 136 * process, before the regular test is run as in the base class [TestCase]. |
| 139 * If the compilation command fails, then the rest of the test is not run. | 137 * If the compilation command fails, then the rest of the test is not run. |
| 140 */ | 138 */ |
| 141 class BrowserTestCase extends TestCase { | 139 class BrowserTestCase extends TestCase { |
| 142 /** | 140 /** |
| (...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 281 void testComplete(int exitCode) { | 279 void testComplete(int exitCode) { |
| 282 new TestOutput(testCase, exitCode, timedOut, stdout, | 280 new TestOutput(testCase, exitCode, timedOut, stdout, |
| 283 stderr, new Date.now().difference(startTime)); | 281 stderr, new Date.now().difference(startTime)); |
| 284 timeoutTimer.cancel(); | 282 timeoutTimer.cancel(); |
| 285 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { | 283 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { |
| 286 print(testCase.displayName); | 284 print(testCase.displayName); |
| 287 for (var line in testCase.output.stderr) print(line); | 285 for (var line in testCase.output.stderr) print(line); |
| 288 for (var line in testCase.output.stdout) print(line); | 286 for (var line in testCase.output.stdout) print(line); |
| 289 } | 287 } |
| 290 if (allowRetries != null && allowRetries | 288 if (allowRetries != null && allowRetries |
| 291 && testCase.usesWebDriver && testCase.output.unexpectedOutput | 289 && testCase.configuration['component'] == 'webdriver' && |
| 292 && testCase.numRetries > 0) { | 290 testCase.output.unexpectedOutput && testCase.numRetries > 0) { |
| 293 // Selenium tests can be flaky. Try rerunning. | 291 // Selenium tests can be flaky. Try rerunning. |
| 294 testCase.output.requestRetry = true; | 292 testCase.output.requestRetry = true; |
| 295 } | 293 } |
| 296 if (testCase.output.requestRetry) { | 294 if (testCase.output.requestRetry) { |
| 297 testCase.output.requestRetry = false; | 295 testCase.output.requestRetry = false; |
| 298 this.timedOut = false; | 296 this.timedOut = false; |
| 299 testCase.dynamic.numRetries--; | 297 testCase.dynamic.numRetries--; |
| 300 print("Potential flake. " + | 298 print("Potential flake. " + |
| 301 "Re-running ${testCase.displayName} " + | 299 "Re-running ${testCase.displayName} " + |
| 302 "(${testCase.dynamic.numRetries} attempt(s) remains)"); | 300 "(${testCase.dynamic.numRetries} attempt(s) remains)"); |
| (...skipping 13 matching lines...) Expand all Loading... |
| 316 int totalSteps = testCase.commands.length; | 314 int totalSteps = testCase.commands.length; |
| 317 String suffix =' (step $currentStep of $totalSteps)'; | 315 String suffix =' (step $currentStep of $totalSteps)'; |
| 318 if (currentStep == totalSteps) { // done with test command | 316 if (currentStep == totalSteps) { // done with test command |
| 319 testComplete(exitCode); | 317 testComplete(exitCode); |
| 320 } else if (exitCode != 0) { | 318 } else if (exitCode != 0) { |
| 321 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 319 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
| 322 testComplete(exitCode); | 320 testComplete(exitCode); |
| 323 } else { | 321 } else { |
| 324 stderr.add('test.dart: Compilion finished $suffix\n'); | 322 stderr.add('test.dart: Compilion finished $suffix\n'); |
| 325 stdout.add('test.dart: Compilion finished $suffix\n'); | 323 stdout.add('test.dart: Compilion finished $suffix\n'); |
| 326 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && | 324 if (currentStep == totalSteps - 1 |
| 327 !testCase.configuration['noBatch']) { | 325 && testCase.configuration['component'] == 'webdriver') { |
| 328 // Note: processQueue will always be non-null for component == webdriver | 326 // Note: processQueue will always be non-null for component == webdriver |
| 329 // (It is only null for component == vm) | 327 // (It is only null for component == vm) |
| 330 processQueue._getBatchRunner(testCase).startTest(testCase); | 328 processQueue._getBatchRunner(testCase).startTest(testCase); |
| 331 } else { | 329 } else { |
| 332 runCommand(testCase.commands[currentStep++], stepExitHandler); | 330 runCommand(testCase.commands[currentStep++], stepExitHandler); |
| 333 } | 331 } |
| 334 } | 332 } |
| 335 } | 333 } |
| 336 | 334 |
| 337 Function makeReadHandler(StringInputStream source, List<String> destination) { | 335 Function makeReadHandler(StringInputStream source, List<String> destination) { |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 395 bool _stdoutDrained = false; | 393 bool _stdoutDrained = false; |
| 396 bool _stderrDrained = false; | 394 bool _stderrDrained = false; |
| 397 Date _startTime; | 395 Date _startTime; |
| 398 Timer _timer; | 396 Timer _timer; |
| 399 | 397 |
| 400 bool _isWebDriver; | 398 bool _isWebDriver; |
| 401 | 399 |
| 402 BatchRunnerProcess(TestCase testCase) { | 400 BatchRunnerProcess(TestCase testCase) { |
| 403 _executable = testCase.commands.last().executable; | 401 _executable = testCase.commands.last().executable; |
| 404 _batchArguments = testCase.batchRunnerArguments; | 402 _batchArguments = testCase.batchRunnerArguments; |
| 405 _isWebDriver = testCase.usesWebDriver; | 403 _isWebDriver = testCase.configuration['component'] == 'webdriver'; |
| 406 } | 404 } |
| 407 | 405 |
| 408 bool get active() => _currentTest != null; | 406 bool get active() => _currentTest != null; |
| 409 | 407 |
| 410 void startTest(TestCase testCase) { | 408 void startTest(TestCase testCase) { |
| 411 _currentTest = testCase; | 409 _currentTest = testCase; |
| 412 if (_process === null) { | 410 if (_process === null) { |
| 413 // Start process if not yet started. | 411 // Start process if not yet started. |
| 414 _executable = testCase.commands.last().executable; | 412 _executable = testCase.commands.last().executable; |
| 415 _startProcess(() { | 413 _startProcess(() { |
| (...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 600 Map<String, List<BatchRunnerProcess>> _batchProcesses; | 598 Map<String, List<BatchRunnerProcess>> _batchProcesses; |
| 601 | 599 |
| 602 // Cache information about test cases per test suite. For multiple | 600 // Cache information about test cases per test suite. For multiple |
| 603 // configurations there is no need to repeatedly search the file | 601 // configurations there is no need to repeatedly search the file |
| 604 // system, generate tests, and search test files for options. | 602 // system, generate tests, and search test files for options. |
| 605 Map<String, List<TestInformation>> _testCache; | 603 Map<String, List<TestInformation>> _testCache; |
| 606 /** | 604 /** |
| 607 * String indicating the browser used to run the tests. Empty if no browser | 605 * String indicating the browser used to run the tests. Empty if no browser |
| 608 * used. | 606 * used. |
| 609 */ | 607 */ |
| 610 String browserUsed = ''; | 608 String browserUsed; |
| 611 /** | |
| 612 * Process running the selenium server .jar (only used for Safari and Opera | |
| 613 * tests.) | |
| 614 */ | |
| 615 Process _seleniumServer = null; | |
| 616 /** True if we are in the process of starting the server. */ | |
| 617 bool _startingServer = false; | |
| 618 | 609 |
| 619 ProcessQueue(int this._maxProcesses, | 610 ProcessQueue(int this._maxProcesses, |
| 620 String progress, | 611 String progress, |
| 621 Date startTime, | 612 Date startTime, |
| 622 bool printTiming, | 613 bool printTiming, |
| 623 Function this._enqueueMoreWork, | 614 Function this._enqueueMoreWork, |
| 624 [bool this._verbose = false, | 615 [bool this._verbose = false, |
| 625 bool this._listTests = false, | 616 bool this._listTests = false, |
| 626 bool this._keepGeneratedTests = false]) | 617 bool this._keepGeneratedTests = false]) |
| 627 : _tests = new Queue<TestCase>(), | 618 : _tests = new Queue<TestCase>(), |
| 628 _progress = new ProgressIndicator.fromName(progress, | 619 _progress = new ProgressIndicator.fromName(progress, |
| 629 startTime, | 620 startTime, |
| 630 printTiming), | 621 printTiming), |
| 631 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), | 622 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), |
| 632 _testCache = new Map<String, List<TestInformation>>() { | 623 _testCache = new Map<String, List<TestInformation>>() { |
| 633 if (!_enqueueMoreWork(this)) _progress.allDone(); | 624 if (!_enqueueMoreWork(this)) _progress.allDone(); |
| 625 browserUsed = ''; |
| 634 } | 626 } |
| 635 | 627 |
| 636 /** | 628 /** |
| 637 * Registers a TestSuite so that all of its tests will be run. | 629 * Registers a TestSuite so that all of its tests will be run. |
| 638 */ | 630 */ |
| 639 void addTestSuite(TestSuite testSuite) { | 631 void addTestSuite(TestSuite testSuite) { |
| 640 _activeTestListers++; | 632 _activeTestListers++; |
| 641 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory, | 633 testSuite.forEachTest(_runTest, _testCache, globalTemporaryDirectory, |
| 642 _testListerDone); | 634 _testListerDone); |
| 643 } | 635 } |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 699 } | 691 } |
| 700 } | 692 } |
| 701 | 693 |
| 702 /** | 694 /** |
| 703 * Perform any cleanup needed once all tests in a TestSuite have completed | 695 * Perform any cleanup needed once all tests in a TestSuite have completed |
| 704 * and notify our progress indicator that we are done. | 696 * and notify our progress indicator that we are done. |
| 705 */ | 697 */ |
| 706 void _cleanupAndMarkDone() { | 698 void _cleanupAndMarkDone() { |
| 707 if (browserUsed != '') { | 699 if (browserUsed != '') { |
| 708 killZombieBrowsers(); | 700 killZombieBrowsers(); |
| 709 if (_isSeleniumAvailable) { | |
| 710 _seleniumServer.kill(); | |
| 711 } | |
| 712 } else { | 701 } else { |
| 713 _progress.allDone(); | 702 _progress.allDone(); |
| 714 } | 703 } |
| 715 } | 704 } |
| 716 | 705 |
| 717 void _checkDone() { | 706 void _checkDone() { |
| 718 // When there are no more active test listers ask for more work | 707 // When there are no more active test listers ask for more work |
| 719 // from process queue users. | 708 // from process queue users. |
| 720 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { | 709 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { |
| 721 _progress.allTestsKnown(); | 710 _progress.allTestsKnown(); |
| (...skipping 18 matching lines...) Expand all Loading... |
| 740 } | 729 } |
| 741 } else { | 730 } else { |
| 742 print('\nDeletion of temp dir $_temporaryDirectory failed.'); | 731 print('\nDeletion of temp dir $_temporaryDirectory failed.'); |
| 743 } | 732 } |
| 744 _cleanupAndMarkDone(); | 733 _cleanupAndMarkDone(); |
| 745 }; | 734 }; |
| 746 } | 735 } |
| 747 } | 736 } |
| 748 } | 737 } |
| 749 } | 738 } |
| 750 | |
| 751 /** | |
| 752 * True if we are using a browser + platform combination that needs the | |
| 753 * Selenium server jar. | |
| 754 */ | |
| 755 bool get _needsSelenium() => new Platform().operatingSystem() == 'macos' && | |
| 756 browserUsed == 'safari'; | |
| 757 | |
| 758 /** True if the Selenium Server is ready to be used. */ | |
| 759 bool get _isSeleniumAvailable() => _seleniumServer != null; | |
| 760 | |
| 761 /** Start the Selenium Server jar, if appropriate for this platform. */ | |
| 762 void _ensureSeleniumServerRunning() { | |
| 763 if (!_isSeleniumAvailable && _startingServer == false) { | |
| 764 _startingServer = true; | |
| 765 _startSeleniumServer(); | |
| 766 } | |
| 767 } | |
| 768 | 739 |
| 769 void _runTest(TestCase test) { | 740 void _runTest(TestCase test) { |
| 770 if (test.usesWebDriver) { | 741 if (test.configuration['component'] == 'webdriver') { |
| 771 browserUsed = test.configuration['browser']; | 742 browserUsed = test.configuration['browser']; |
| 772 if (_needsSelenium) _ensureSeleniumServerRunning(); | |
| 773 } | 743 } |
| 774 _progress.testAdded(); | 744 _progress.testAdded(); |
| 775 _tests.add(test); | 745 _tests.add(test); |
| 776 _tryRunTest(); | 746 _tryRunTest(); |
| 777 } | 747 } |
| 778 | 748 |
| 779 /** | |
| 780 * Monitor the output of the Selenium server, to know when we are ready to | |
| 781 * begin running tests. | |
| 782 * source: Output(Stream) from the Java server. | |
| 783 */ | |
| 784 Function makeSeleniumServerHandler(StringInputStream source) { | |
| 785 return () { | |
| 786 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. | |
| 787 var line = source.readLine(); | |
| 788 while (null != line) { | |
| 789 if (const RegExp(@".*Started.*Server.*").hasMatch(line) || | |
| 790 const RegExp(@"Exception.*Selenium is already running.*").hasMatch( | |
| 791 line)) { | |
| 792 for (int i = 0; i < _maxProcesses; i++) { | |
| 793 // Restart all the processes that have been waiting/stopped for | |
| 794 // the server to start up. If we just call this once we end up | |
| 795 // with a single-"threaded" run. | |
| 796 _tryRunTest(); | |
| 797 } | |
| 798 } | |
| 799 line = source.readLine(); | |
| 800 } | |
| 801 }; | |
| 802 } | |
| 803 | |
| 804 /** | |
| 805 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 | |
| 806 * Java server. | |
| 807 */ | |
| 808 void _startSeleniumServer() { | |
| 809 // Get the absolute path to the Selenium jar. | |
| 810 String filePath = new Options().script; | |
| 811 String pathSep = new Platform().pathSeparator(); | |
| 812 int index = filePath.lastIndexOf(pathSep); | |
| 813 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; | |
| 814 var dir = new Directory(filePath); | |
| 815 dir.onFile = (String file) { | |
| 816 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) | |
| 817 && _seleniumServer == null) { | |
| 818 _seleniumServer = new Process.start('java', ['-jar', file]); | |
| 819 // Heads up: there seems to an obscure data race of some form in | |
| 820 // the VM between launching the server process and launching the test | |
| 821 // tasks that disappears when you read IO (which is convenient, since | |
| 822 // that is our condition for knowing that the server is ready). | |
| 823 StringInputStream stdoutStringStream = | |
| 824 new StringInputStream(_seleniumServer.stdout); | |
| 825 StringInputStream stderrStringStream = | |
| 826 new StringInputStream(_seleniumServer.stderr); | |
| 827 stdoutStringStream.onLine = | |
| 828 makeSeleniumServerHandler(stdoutStringStream); | |
| 829 stderrStringStream.onLine = | |
| 830 makeSeleniumServerHandler(stderrStringStream); | |
| 831 } | |
| 832 }; | |
| 833 dir.list(); | |
| 834 } | |
| 835 | |
| 836 void _terminateBatchRunners() { | 749 void _terminateBatchRunners() { |
| 837 for (var runners in _batchProcesses.getValues()) { | 750 for (var runners in _batchProcesses.getValues()) { |
| 838 for (var runner in runners) { | 751 for (var runner in runners) { |
| 839 runner.terminate(); | 752 runner.terminate(); |
| 840 } | 753 } |
| 841 } | 754 } |
| 842 } | 755 } |
| 843 | 756 |
| 844 BatchRunnerProcess _getBatchRunner(TestCase test) { | 757 BatchRunnerProcess _getBatchRunner(TestCase test) { |
| 845 // Start batch processes if needed | 758 // Start batch processes if needed |
| (...skipping 19 matching lines...) Expand all Loading... |
| 865 TestCase test = _tests.removeFirst(); | 778 TestCase test = _tests.removeFirst(); |
| 866 if (_verbose) print(test.commands.last().commandLine); | 779 if (_verbose) print(test.commands.last().commandLine); |
| 867 if (_listTests) { | 780 if (_listTests) { |
| 868 final String tab = '\t'; | 781 final String tab = '\t'; |
| 869 String outcomes = | 782 String outcomes = |
| 870 Strings.join(new List.from(test.expectedOutcomes), ','); | 783 Strings.join(new List.from(test.expectedOutcomes), ','); |
| 871 print(test.displayName + tab + outcomes + tab + test.isNegative + | 784 print(test.displayName + tab + outcomes + tab + test.isNegative + |
| 872 tab + Strings.join(test.commands.last().arguments, tab)); | 785 tab + Strings.join(test.commands.last().arguments, tab)); |
| 873 return; | 786 return; |
| 874 } | 787 } |
| 875 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { | |
| 876 // The server is not ready to run Selenium tests. Put them back in the | |
| 877 // queue. | |
| 878 _tests.addFirst(test); | |
| 879 return; | |
| 880 } | |
| 881 _progress.start(test); | 788 _progress.start(test); |
| 882 Function oldCallback = test.completedHandler; | 789 Function oldCallback = test.completedHandler; |
| 883 Function wrapper = (TestCase test_arg) { | 790 Function wrapper = (TestCase test_arg) { |
| 884 _numProcesses--; | 791 _numProcesses--; |
| 885 _progress.done(test_arg); | 792 _progress.done(test_arg); |
| 886 _tryRunTest(); | 793 _tryRunTest(); |
| 887 oldCallback(test_arg); | 794 oldCallback(test_arg); |
| 888 }; | 795 }; |
| 889 test.completedHandler = wrapper; | 796 test.completedHandler = wrapper; |
| 890 if (test.configuration['component'] == 'dartc' && | 797 if (test.configuration['component'] == 'dartc' && |
| 891 test.displayName != 'dartc/junit_tests') { | 798 test.displayName != 'dartc/junit_tests') { |
| 892 _getBatchRunner(test).startTest(test); | 799 _getBatchRunner(test).startTest(test); |
| 893 } else { | 800 } else { |
| 894 // Once we've actually failed a test, technically, we wouldn't need to | 801 // Once we've actually failed a test, technically, we wouldn't need to |
| 895 // bother retrying any subsequent tests since the bot is already red. | 802 // bother retrying any subsequent tests since the bot is already red. |
| 896 // However, we continue to retry tests until we have actually failed | 803 // However, we continue to retry tests until we have actually failed |
| 897 // four tests (arbitrarily chosen) for more debugable output, so that | 804 // four tests (arbitrarily chosen) for more debugable output, so that |
| 898 // the developer doesn't waste his or her time trying to fix a bunch of | 805 // the developer doesn't waste his or her time trying to fix a bunch of |
| 899 // tests that appear to be broken but were actually just flakes that | 806 // tests that appear to be broken but were actually just flakes that |
| 900 // didn't get retried because there had already been one failure. | 807 // didn't get retried because there had already been one failure. |
| 901 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 808 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 902 new RunningProcess(test, allowRetry, this).start(); | 809 new RunningProcess(test, allowRetry, this).start(); |
| 903 } | 810 } |
| 904 _numProcesses++; | 811 _numProcesses++; |
| 905 } | 812 } |
| 906 } | 813 } |
| 907 } | 814 } |
| OLD | NEW |