| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), | 117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), |
| 118 newCommand.commandLine); | 118 newCommand.commandLine); |
| 119 } | 119 } |
| 120 commands = newCommands; | 120 commands = newCommands; |
| 121 } | 121 } |
| 122 } | 122 } |
| 123 | 123 |
| 124 int get timeout() => configuration['timeout']; | 124 int get timeout() => configuration['timeout']; |
| 125 | 125 |
| 126 String get configurationString() { | 126 String get configurationString() { |
| 127 final component = configuration['component']; | 127 final compiler = configuration['compiler']; |
| 128 final runtime = configuration['runtime']; |
| 128 final mode = configuration['mode']; | 129 final mode = configuration['mode']; |
| 129 final arch = configuration['arch']; | 130 final arch = configuration['arch']; |
| 130 return "$component ${mode}_$arch"; | 131 return "$compiler-$runtime ${mode}_$arch"; |
| 131 } | 132 } |
| 132 | 133 |
| 133 List<String> get batchRunnerArguments() => ['-batch']; | 134 List<String> get batchRunnerArguments() => ['-batch']; |
| 134 List<String> get batchTestArguments() => commands.last().arguments; | 135 List<String> get batchTestArguments() => commands.last().arguments; |
| 135 | 136 |
| 136 void completed() { completedHandler(this); } | 137 void completed() { completedHandler(this); } |
| 137 | 138 |
| 138 bool get usesWebDriver() => configuration['component'] == 'webdriver'; | 139 bool get usesWebDriver() => (const ['chrome', 'ff', 'safari', 'ie', 'opera']) |
| 140 .indexOf(configuration['runtime']) >= 0; |
| 139 } | 141 } |
| 140 | 142 |
| 141 | 143 |
| 142 /** | 144 /** |
| 143 * BrowserTestCase has an extra compilation command that is run in a separate | 145 * BrowserTestCase has an extra compilation command that is run in a separate |
| 144 * process, before the regular test is run as in the base class [TestCase]. | 146 * process, before the regular test is run as in the base class [TestCase]. |
| 145 * If the compilation command fails, then the rest of the test is not run. | 147 * If the compilation command fails, then the rest of the test is not run. |
| 146 */ | 148 */ |
| 147 class BrowserTestCase extends TestCase { | 149 class BrowserTestCase extends TestCase { |
| 148 /** | 150 /** |
| (...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 222 Duration this.time) { | 224 Duration this.time) { |
| 223 testCase.output = this; | 225 testCase.output = this; |
| 224 diagnostics = []; | 226 diagnostics = []; |
| 225 } | 227 } |
| 226 | 228 |
| 227 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu
t, | 229 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu
t, |
| 228 List<String> stdout, List<String> stderr, Dur
ation time) { | 230 List<String> stdout, List<String> stderr, Dur
ation time) { |
| 229 if (testCase is BrowserTestCase) { | 231 if (testCase is BrowserTestCase) { |
| 230 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, | 232 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, |
| 231 stdout, stderr, time); | 233 stdout, stderr, time); |
| 232 } else if (testCase.configuration['component'] == 'dartc') { | 234 } else if (testCase.configuration['compiler'] == 'dartc') { |
| 233 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, | 235 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, |
| 234 stdout, stderr, time); | 236 stdout, stderr, time); |
| 235 } | 237 } |
| 236 return new TestOutputImpl(testCase, exitCode, timedOut, | 238 return new TestOutputImpl(testCase, exitCode, timedOut, |
| 237 stdout, stderr, time); | 239 stdout, stderr, time); |
| 238 } | 240 } |
| 239 | 241 |
| 240 String get result() => | 242 String get result() => |
| 241 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); | 243 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); |
| 242 | 244 |
| (...skipping 290 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 533 if (currentStep == totalSteps) { // done with test command | 535 if (currentStep == totalSteps) { // done with test command |
| 534 testComplete(exitCode); | 536 testComplete(exitCode); |
| 535 } else if (exitCode != 0) { | 537 } else if (exitCode != 0) { |
| 536 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 538 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
| 537 testComplete(exitCode); | 539 testComplete(exitCode); |
| 538 } else { | 540 } else { |
| 539 stderr.add('test.dart: Compilion finished $suffix\n'); | 541 stderr.add('test.dart: Compilion finished $suffix\n'); |
| 540 stdout.add('test.dart: Compilion finished $suffix\n'); | 542 stdout.add('test.dart: Compilion finished $suffix\n'); |
| 541 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && | 543 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && |
| 542 !testCase.configuration['noBatch']) { | 544 !testCase.configuration['noBatch']) { |
| 543 // Note: processQueue will always be non-null for component == webdriver | 545 // Note: processQueue will always be non-null for runtime == ie, ff, |
| 544 // (It is only null for component == vm) | 546 // safari, chrome, opera. (It is only null for runtime == vm) |
| 545 processQueue._getBatchRunner(testCase).startTest(testCase); | 547 processQueue._getBatchRunner(testCase).startTest(testCase); |
| 546 } else { | 548 } else { |
| 547 runCommand(testCase.commands[currentStep++], stepExitHandler); | 549 runCommand(testCase.commands[currentStep++], stepExitHandler); |
| 548 } | 550 } |
| 549 } | 551 } |
| 550 } | 552 } |
| 551 | 553 |
| 552 Function makeReadHandler(StringInputStream source, List<String> destination) { | 554 Function makeReadHandler(StringInputStream source, List<String> destination) { |
| 553 return () { | 555 return () { |
| 554 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. | 556 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. |
| (...skipping 487 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1042 void _terminateBatchRunners() { | 1044 void _terminateBatchRunners() { |
| 1043 for (var runners in _batchProcesses.getValues()) { | 1045 for (var runners in _batchProcesses.getValues()) { |
| 1044 for (var runner in runners) { | 1046 for (var runner in runners) { |
| 1045 runner.terminate(); | 1047 runner.terminate(); |
| 1046 } | 1048 } |
| 1047 } | 1049 } |
| 1048 } | 1050 } |
| 1049 | 1051 |
| 1050 BatchRunnerProcess _getBatchRunner(TestCase test) { | 1052 BatchRunnerProcess _getBatchRunner(TestCase test) { |
| 1051 // Start batch processes if needed | 1053 // Start batch processes if needed |
| 1052 var component = test.configuration['component']; | 1054 var compiler = test.configuration['compiler']; |
| 1053 var runners = _batchProcesses[component]; | 1055 var runners = _batchProcesses[compiler]; |
| 1054 if (runners == null) { | 1056 if (runners == null) { |
| 1055 runners = new List<BatchRunnerProcess>(_maxProcesses); | 1057 runners = new List<BatchRunnerProcess>(_maxProcesses); |
| 1056 for (int i = 0; i < _maxProcesses; i++) { | 1058 for (int i = 0; i < _maxProcesses; i++) { |
| 1057 runners[i] = new BatchRunnerProcess(test); | 1059 runners[i] = new BatchRunnerProcess(test); |
| 1058 } | 1060 } |
| 1059 _batchProcesses[component] = runners; | 1061 _batchProcesses[compiler] = runners; |
| 1060 } | 1062 } |
| 1061 | 1063 |
| 1062 for (var runner in runners) { | 1064 for (var runner in runners) { |
| 1063 if (!runner.active) return runner; | 1065 if (!runner.active) return runner; |
| 1064 } | 1066 } |
| 1065 throw new Exception('Unable to find inactive batch runner.'); | 1067 throw new Exception('Unable to find inactive batch runner.'); |
| 1066 } | 1068 } |
| 1067 | 1069 |
| 1068 void _tryRunTest() { | 1070 void _tryRunTest() { |
| 1069 _checkDone(); | 1071 _checkDone(); |
| (...skipping 22 matching lines...) Expand all Loading... |
| 1092 } | 1094 } |
| 1093 _progress.start(test); | 1095 _progress.start(test); |
| 1094 Function oldCallback = test.completedHandler; | 1096 Function oldCallback = test.completedHandler; |
| 1095 Function wrapper = (TestCase test_arg) { | 1097 Function wrapper = (TestCase test_arg) { |
| 1096 _numProcesses--; | 1098 _numProcesses--; |
| 1097 _progress.done(test_arg); | 1099 _progress.done(test_arg); |
| 1098 _tryRunTest(); | 1100 _tryRunTest(); |
| 1099 oldCallback(test_arg); | 1101 oldCallback(test_arg); |
| 1100 }; | 1102 }; |
| 1101 test.completedHandler = wrapper; | 1103 test.completedHandler = wrapper; |
| 1102 if (test.configuration['component'] == 'dartc' && | 1104 if (test.configuration['compiler'] == 'dartc' && |
| 1103 test.displayName != 'dartc/junit_tests') { | 1105 test.displayName != 'dartc/junit_tests') { |
| 1104 _getBatchRunner(test).startTest(test); | 1106 _getBatchRunner(test).startTest(test); |
| 1105 } else { | 1107 } else { |
| 1106 // Once we've actually failed a test, technically, we wouldn't need to | 1108 // Once we've actually failed a test, technically, we wouldn't need to |
| 1107 // bother retrying any subsequent tests since the bot is already red. | 1109 // bother retrying any subsequent tests since the bot is already red. |
| 1108 // However, we continue to retry tests until we have actually failed | 1110 // However, we continue to retry tests until we have actually failed |
| 1109 // four tests (arbitrarily chosen) for more debugable output, so that | 1111 // four tests (arbitrarily chosen) for more debugable output, so that |
| 1110 // the developer doesn't waste his or her time trying to fix a bunch of | 1112 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1111 // tests that appear to be broken but were actually just flakes that | 1113 // tests that appear to be broken but were actually just flakes that |
| 1112 // didn't get retried because there had already been one failure. | 1114 // didn't get retried because there had already been one failure. |
| 1113 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1115 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1114 new RunningProcess(test, allowRetry, this).start(); | 1116 new RunningProcess(test, allowRetry, this).start(); |
| 1115 } | 1117 } |
| 1116 _numProcesses++; | 1118 _numProcesses++; |
| 1117 } | 1119 } |
| 1118 } | 1120 } |
| 1119 } | 1121 } |
| OLD | NEW |