OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), | 117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), |
118 newCommand.commandLine); | 118 newCommand.commandLine); |
119 } | 119 } |
120 commands = newCommands; | 120 commands = newCommands; |
121 } | 121 } |
122 } | 122 } |
123 | 123 |
124 int get timeout() => configuration['timeout']; | 124 int get timeout() => configuration['timeout']; |
125 | 125 |
126 String get configurationString() { | 126 String get configurationString() { |
127 final component = configuration['component']; | 127 final compiler = configuration['compiler']; |
| 128 final runtime = configuration['runtime']; |
128 final mode = configuration['mode']; | 129 final mode = configuration['mode']; |
129 final arch = configuration['arch']; | 130 final arch = configuration['arch']; |
130 return "$component ${mode}_$arch"; | 131 return "$compiler-$runtime ${mode}_$arch"; |
131 } | 132 } |
132 | 133 |
133 List<String> get batchRunnerArguments() => ['-batch']; | 134 List<String> get batchRunnerArguments() => ['-batch']; |
134 List<String> get batchTestArguments() => commands.last().arguments; | 135 List<String> get batchTestArguments() => commands.last().arguments; |
135 | 136 |
136 void completed() { completedHandler(this); } | 137 void completed() { completedHandler(this); } |
137 | 138 |
138 bool get usesWebDriver() => configuration['component'] == 'webdriver'; | 139 bool get usesWebDriver() => (const ['chrome', 'ff', 'safari', 'ie', 'opera']) |
| 140 .indexOf(configuration['runtime']) >= 0; |
139 } | 141 } |
140 | 142 |
141 | 143 |
142 /** | 144 /** |
143 * BrowserTestCase has an extra compilation command that is run in a separate | 145 * BrowserTestCase has an extra compilation command that is run in a separate |
144 * process, before the regular test is run as in the base class [TestCase]. | 146 * process, before the regular test is run as in the base class [TestCase]. |
145 * If the compilation command fails, then the rest of the test is not run. | 147 * If the compilation command fails, then the rest of the test is not run. |
146 */ | 148 */ |
147 class BrowserTestCase extends TestCase { | 149 class BrowserTestCase extends TestCase { |
148 /** | 150 /** |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
222 Duration this.time) { | 224 Duration this.time) { |
223 testCase.output = this; | 225 testCase.output = this; |
224 diagnostics = []; | 226 diagnostics = []; |
225 } | 227 } |
226 | 228 |
227 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu
t, | 229 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu
t, |
228 List<String> stdout, List<String> stderr, Dur
ation time) { | 230 List<String> stdout, List<String> stderr, Dur
ation time) { |
229 if (testCase is BrowserTestCase) { | 231 if (testCase is BrowserTestCase) { |
230 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, | 232 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, |
231 stdout, stderr, time); | 233 stdout, stderr, time); |
232 } else if (testCase.configuration['component'] == 'dartc') { | 234 } else if (testCase.configuration['compiler'] == 'dartc') { |
233 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, | 235 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, |
234 stdout, stderr, time); | 236 stdout, stderr, time); |
235 } | 237 } |
236 return new TestOutputImpl(testCase, exitCode, timedOut, | 238 return new TestOutputImpl(testCase, exitCode, timedOut, |
237 stdout, stderr, time); | 239 stdout, stderr, time); |
238 } | 240 } |
239 | 241 |
240 String get result() => | 242 String get result() => |
241 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); | 243 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); |
242 | 244 |
(...skipping 290 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
533 if (currentStep == totalSteps) { // done with test command | 535 if (currentStep == totalSteps) { // done with test command |
534 testComplete(exitCode); | 536 testComplete(exitCode); |
535 } else if (exitCode != 0) { | 537 } else if (exitCode != 0) { |
536 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 538 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
537 testComplete(exitCode); | 539 testComplete(exitCode); |
538 } else { | 540 } else { |
539 stderr.add('test.dart: Compilion finished $suffix\n'); | 541 stderr.add('test.dart: Compilion finished $suffix\n'); |
540 stdout.add('test.dart: Compilion finished $suffix\n'); | 542 stdout.add('test.dart: Compilion finished $suffix\n'); |
541 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && | 543 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && |
542 !testCase.configuration['noBatch']) { | 544 !testCase.configuration['noBatch']) { |
543 // Note: processQueue will always be non-null for component == webdriver | 545 // Note: processQueue will always be non-null for runtime == ie, ff, |
544 // (It is only null for component == vm) | 546 // safari, chrome, opera. (It is only null for runtime == vm) |
545 processQueue._getBatchRunner(testCase).startTest(testCase); | 547 processQueue._getBatchRunner(testCase).startTest(testCase); |
546 } else { | 548 } else { |
547 runCommand(testCase.commands[currentStep++], stepExitHandler); | 549 runCommand(testCase.commands[currentStep++], stepExitHandler); |
548 } | 550 } |
549 } | 551 } |
550 } | 552 } |
551 | 553 |
552 Function makeReadHandler(StringInputStream source, List<String> destination) { | 554 Function makeReadHandler(StringInputStream source, List<String> destination) { |
553 return () { | 555 return () { |
554 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. | 556 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. |
(...skipping 487 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1042 void _terminateBatchRunners() { | 1044 void _terminateBatchRunners() { |
1043 for (var runners in _batchProcesses.getValues()) { | 1045 for (var runners in _batchProcesses.getValues()) { |
1044 for (var runner in runners) { | 1046 for (var runner in runners) { |
1045 runner.terminate(); | 1047 runner.terminate(); |
1046 } | 1048 } |
1047 } | 1049 } |
1048 } | 1050 } |
1049 | 1051 |
1050 BatchRunnerProcess _getBatchRunner(TestCase test) { | 1052 BatchRunnerProcess _getBatchRunner(TestCase test) { |
1051 // Start batch processes if needed | 1053 // Start batch processes if needed |
1052 var component = test.configuration['component']; | 1054 var compiler = test.configuration['compiler']; |
1053 var runners = _batchProcesses[component]; | 1055 var runners = _batchProcesses[compiler]; |
1054 if (runners == null) { | 1056 if (runners == null) { |
1055 runners = new List<BatchRunnerProcess>(_maxProcesses); | 1057 runners = new List<BatchRunnerProcess>(_maxProcesses); |
1056 for (int i = 0; i < _maxProcesses; i++) { | 1058 for (int i = 0; i < _maxProcesses; i++) { |
1057 runners[i] = new BatchRunnerProcess(test); | 1059 runners[i] = new BatchRunnerProcess(test); |
1058 } | 1060 } |
1059 _batchProcesses[component] = runners; | 1061 _batchProcesses[compiler] = runners; |
1060 } | 1062 } |
1061 | 1063 |
1062 for (var runner in runners) { | 1064 for (var runner in runners) { |
1063 if (!runner.active) return runner; | 1065 if (!runner.active) return runner; |
1064 } | 1066 } |
1065 throw new Exception('Unable to find inactive batch runner.'); | 1067 throw new Exception('Unable to find inactive batch runner.'); |
1066 } | 1068 } |
1067 | 1069 |
1068 void _tryRunTest() { | 1070 void _tryRunTest() { |
1069 _checkDone(); | 1071 _checkDone(); |
(...skipping 22 matching lines...) Expand all Loading... |
1092 } | 1094 } |
1093 _progress.start(test); | 1095 _progress.start(test); |
1094 Function oldCallback = test.completedHandler; | 1096 Function oldCallback = test.completedHandler; |
1095 Function wrapper = (TestCase test_arg) { | 1097 Function wrapper = (TestCase test_arg) { |
1096 _numProcesses--; | 1098 _numProcesses--; |
1097 _progress.done(test_arg); | 1099 _progress.done(test_arg); |
1098 _tryRunTest(); | 1100 _tryRunTest(); |
1099 oldCallback(test_arg); | 1101 oldCallback(test_arg); |
1100 }; | 1102 }; |
1101 test.completedHandler = wrapper; | 1103 test.completedHandler = wrapper; |
1102 if (test.configuration['component'] == 'dartc' && | 1104 if (test.configuration['compiler'] == 'dartc' && |
1103 test.displayName != 'dartc/junit_tests') { | 1105 test.displayName != 'dartc/junit_tests') { |
1104 _getBatchRunner(test).startTest(test); | 1106 _getBatchRunner(test).startTest(test); |
1105 } else { | 1107 } else { |
1106 // Once we've actually failed a test, technically, we wouldn't need to | 1108 // Once we've actually failed a test, technically, we wouldn't need to |
1107 // bother retrying any subsequent tests since the bot is already red. | 1109 // bother retrying any subsequent tests since the bot is already red. |
1108 // However, we continue to retry tests until we have actually failed | 1110 // However, we continue to retry tests until we have actually failed |
1109 // four tests (arbitrarily chosen) for more debugable output, so that | 1111 // four tests (arbitrarily chosen) for more debugable output, so that |
1110 // the developer doesn't waste his or her time trying to fix a bunch of | 1112 // the developer doesn't waste his or her time trying to fix a bunch of |
1111 // tests that appear to be broken but were actually just flakes that | 1113 // tests that appear to be broken but were actually just flakes that |
1112 // didn't get retried because there had already been one failure. | 1114 // didn't get retried because there had already been one failure. |
1113 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1115 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
1114 new RunningProcess(test, allowRetry, this).start(); | 1116 new RunningProcess(test, allowRetry, this).start(); |
1115 } | 1117 } |
1116 _numProcesses++; | 1118 _numProcesses++; |
1117 } | 1119 } |
1118 } | 1120 } |
1119 } | 1121 } |
OLD | NEW |