| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 444 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 455 class RunningProcess { | 455 class RunningProcess { |
| 456 ProcessQueue processQueue; | 456 ProcessQueue processQueue; |
| 457 Process process; | 457 Process process; |
| 458 TestCase testCase; | 458 TestCase testCase; |
| 459 bool timedOut = false; | 459 bool timedOut = false; |
| 460 Date startTime; | 460 Date startTime; |
| 461 Timer timeoutTimer; | 461 Timer timeoutTimer; |
| 462 List<String> stdout; | 462 List<String> stdout; |
| 463 List<String> stderr; | 463 List<String> stderr; |
| 464 List<Function> handlers; | 464 List<Function> handlers; |
| 465 bool allowRetries = false; | 465 bool allowRetries; |
| 466 | 466 |
| 467 /** Which command of [testCase.commands] is currently being executed. */ | 467 /** Which command of [testCase.commands] is currently being executed. */ |
| 468 int currentStep; | 468 int currentStep; |
| 469 | 469 |
| 470 RunningProcess(TestCase this.testCase, | 470 RunningProcess(TestCase this.testCase, |
| 471 [this.allowRetries, this.processQueue]); | 471 [this.allowRetries = false, this.processQueue]); |
| 472 | 472 |
| 473 /** | 473 /** |
| 474 * Called when all commands are executed. [exitCode] is 0 if all command | 474 * Called when all commands are executed. [exitCode] is 0 if all command |
| 475 * succeded, otherwise it will have the exit code of the first failing | 475 * succeded, otherwise it will have the exit code of the first failing |
| 476 * command. | 476 * command. |
| 477 */ | 477 */ |
| 478 void testComplete(int exitCode) { | 478 void testComplete(int exitCode) { |
| 479 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, | 479 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, |
| 480 stderr, new Date.now().difference(startTime)); | 480 stderr, new Date.now().difference(startTime)); |
| 481 timeoutTimer.cancel(); | 481 timeoutTimer.cancel(); |
| 482 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { | 482 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { |
| 483 print(testCase.displayName); | 483 print(testCase.displayName); |
| 484 for (var line in testCase.output.stderr) print(line); | 484 for (var line in testCase.output.stderr) print(line); |
| 485 for (var line in testCase.output.stdout) print(line); | 485 for (var line in testCase.output.stdout) print(line); |
| 486 } | 486 } |
| 487 if (allowRetries != null && allowRetries | 487 if (allowRetries && testCase.usesWebDriver |
| 488 && testCase.usesWebDriver && testCase.output.unexpectedOutput | 488 && testCase.output.unexpectedOutput && testCase.numRetries > 0) { |
| 489 && testCase.numRetries > 0) { | |
| 490 // Selenium tests can be flaky. Try rerunning. | 489 // Selenium tests can be flaky. Try rerunning. |
| 491 testCase.output.requestRetry = true; | 490 testCase.output.requestRetry = true; |
| 492 } | 491 } |
| 493 if (testCase.output.requestRetry) { | 492 if (testCase.output.requestRetry) { |
| 494 testCase.output.requestRetry = false; | 493 testCase.output.requestRetry = false; |
| 495 this.timedOut = false; | 494 this.timedOut = false; |
| 496 testCase.dynamic.numRetries--; | 495 testCase.dynamic.numRetries--; |
| 497 print("Potential flake. Re-running ${testCase.displayName} " + | 496 print("Potential flake. Re-running ${testCase.displayName} " + |
| 498 "(${testCase.dynamic.numRetries} attempt(s) remains)"); | 497 "(${testCase.dynamic.numRetries} attempt(s) remains)"); |
| 499 this.start(); | 498 this.start(); |
| (...skipping 360 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 860 throw new Exception( | 859 throw new Exception( |
| 861 'Test suite requires temporary directory. Not supported on Windows.'); | 860 'Test suite requires temporary directory. Not supported on Windows.'); |
| 862 } | 861 } |
| 863 var tempDir = new Directory(''); | 862 var tempDir = new Directory(''); |
| 864 tempDir.createTempSync(); | 863 tempDir.createTempSync(); |
| 865 _temporaryDirectory = tempDir.path; | 864 _temporaryDirectory = tempDir.path; |
| 866 return _temporaryDirectory; | 865 return _temporaryDirectory; |
| 867 } | 866 } |
| 868 | 867 |
| 869 /** | 868 /** |
| 870 * Sometimes Webdriver doesn't close every browser window when it's done | |
| 871 * with a test. At the end of all tests we clear out any neglected processes | |
| 872 * that are still running. | |
| 873 */ | |
| 874 void killZombieBrowsers() { | |
| 875 String chromeName = 'chrome'; | |
| 876 if (new Platform().operatingSystem() == 'macos') { | |
| 877 chromeName = 'Google\ Chrome'; | |
| 878 } | |
| 879 Map<String, List<String>> processNames = {'ie': ['iexplore'], | |
| 880 'safari': ['Safari'], 'ff': ['firefox', 'firefox-bin'], | |
| 881 'chrome': ['chromedriver', chromeName]}; | |
| 882 for (String name in processNames[browserUsed]) { | |
| 883 Process process = null; | |
| 884 if (new Platform().operatingSystem() == 'windows') { | |
| 885 process = new Process.start( | |
| 886 'C:\\Windows\\System32\\taskkill.exe', ['/F', '/IM', name + '.exe', | |
| 887 '/T']); | |
| 888 } else { | |
| 889 process = new Process.start('killall', ['-9', name]); | |
| 890 } | |
| 891 | |
| 892 if (name == processNames[browserUsed].last()) { | |
| 893 process.onExit = (exitCode) { | |
| 894 process.close(); | |
| 895 _progress.allDone(); | |
| 896 }; | |
| 897 process.onError = (error) { | |
| 898 _progress.allDone(); | |
| 899 }; | |
| 900 } else { | |
| 901 process.onExit = (exitCode) { | |
| 902 process.close(); | |
| 903 }; | |
| 904 } | |
| 905 } | |
| 906 } | |
| 907 | |
| 908 /** | |
| 909 * Perform any cleanup needed once all tests in a TestSuite have completed | 869 * Perform any cleanup needed once all tests in a TestSuite have completed |
| 910 * and notify our progress indicator that we are done. | 870 * and notify our progress indicator that we are done. |
| 911 */ | 871 */ |
| 912 void _cleanupAndMarkDone() { | 872 void _cleanupAndMarkDone() { |
| 913 if (browserUsed != '' && _progress is BuildbotProgressIndicator) { | 873 if (browserUsed != '' && _seleniumServer != null) { |
| 914 killZombieBrowsers(); | |
| 915 if (_seleniumServer != null) { | |
| 916 _seleniumServer.kill(); | 874 _seleniumServer.kill(); |
| 917 } | |
| 918 } else { | 875 } else { |
| 919 _progress.allDone(); | 876 _progress.allDone(); |
| 920 } | 877 } |
| 921 } | 878 } |
| 922 | 879 |
| 923 void _checkDone() { | 880 void _checkDone() { |
| 924 // When there are no more active test listers ask for more work | 881 // When there are no more active test listers ask for more work |
| 925 // from process queue users. | 882 // from process queue users. |
| 926 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { | 883 if (_activeTestListers == 0 && !_enqueueMoreWork(this)) { |
| 927 _progress.allTestsKnown(); | 884 _progress.allTestsKnown(); |
| (...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1103 print(test.displayName + tab + outcomes + tab + test.isNegative + | 1060 print(test.displayName + tab + outcomes + tab + test.isNegative + |
| 1104 tab + Strings.join(test.commands.last().arguments, tab)); | 1061 tab + Strings.join(test.commands.last().arguments, tab)); |
| 1105 return; | 1062 return; |
| 1106 } | 1063 } |
| 1107 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { | 1064 if (test.usesWebDriver && _needsSelenium && !_isSeleniumAvailable) { |
| 1108 // The server is not ready to run Selenium tests. Put the test back in | 1065 // The server is not ready to run Selenium tests. Put the test back in |
| 1109 // the queue. | 1066 // the queue. |
| 1110 _tests.addFirst(test); | 1067 _tests.addFirst(test); |
| 1111 return; | 1068 return; |
| 1112 } | 1069 } |
| 1113 if (_verbose) print(test.commands.last().commandLine); | 1070 if (_verbose) { |
| 1071 for (Command command in test.commands) print(command.commandLine); |
| 1072 } |
| 1114 _progress.start(test); | 1073 _progress.start(test); |
| 1115 Function oldCallback = test.completedHandler; | 1074 Function oldCallback = test.completedHandler; |
| 1116 Function wrapper = (TestCase test_arg) { | 1075 Function wrapper = (TestCase test_arg) { |
| 1117 _numProcesses--; | 1076 _numProcesses--; |
| 1118 _progress.done(test_arg); | 1077 _progress.done(test_arg); |
| 1119 _tryRunTest(); | 1078 _tryRunTest(); |
| 1120 oldCallback(test_arg); | 1079 oldCallback(test_arg); |
| 1121 }; | 1080 }; |
| 1122 test.completedHandler = wrapper; | 1081 test.completedHandler = wrapper; |
| 1123 if (test.configuration['component'] == 'dartc' && | 1082 if (test.configuration['component'] == 'dartc' && |
| 1124 test.displayName != 'dartc/junit_tests') { | 1083 test.displayName != 'dartc/junit_tests') { |
| 1125 _getBatchRunner(test).startTest(test); | 1084 _getBatchRunner(test).startTest(test); |
| 1126 } else { | 1085 } else { |
| 1127 // Once we've actually failed a test, technically, we wouldn't need to | 1086 // Once we've actually failed a test, technically, we wouldn't need to |
| 1128 // bother retrying any subsequent tests since the bot is already red. | 1087 // bother retrying any subsequent tests since the bot is already red. |
| 1129 // However, we continue to retry tests until we have actually failed | 1088 // However, we continue to retry tests until we have actually failed |
| 1130 // four tests (arbitrarily chosen) for more debugable output, so that | 1089 // four tests (arbitrarily chosen) for more debugable output, so that |
| 1131 // the developer doesn't waste his or her time trying to fix a bunch of | 1090 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1132 // tests that appear to be broken but were actually just flakes that | 1091 // tests that appear to be broken but were actually just flakes that |
| 1133 // didn't get retried because there had already been one failure. | 1092 // didn't get retried because there had already been one failure. |
| 1134 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1093 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1135 new RunningProcess(test, allowRetry, this).start(); | 1094 new RunningProcess(test, allowRetry, this).start(); |
| 1136 } | 1095 } |
| 1137 _numProcesses++; | 1096 _numProcesses++; |
| 1138 } | 1097 } |
| 1139 } | 1098 } |
| 1140 } | 1099 } |
| OLD | NEW |