OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), | 117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), |
118 newCommand.commandLine); | 118 newCommand.commandLine); |
119 } | 119 } |
120 commands = newCommands; | 120 commands = newCommands; |
121 } | 121 } |
122 } | 122 } |
123 | 123 |
124 int get timeout() => configuration['timeout']; | 124 int get timeout() => configuration['timeout']; |
125 | 125 |
126 String get configurationString() { | 126 String get configurationString() { |
127 final component = configuration['component']; | 127 final compiler = configuration['compiler']; |
| 128 final runtime = configuration['runtime']; |
128 final mode = configuration['mode']; | 129 final mode = configuration['mode']; |
129 final arch = configuration['arch']; | 130 final arch = configuration['arch']; |
130 return "$component ${mode}_$arch"; | 131 return "$compiler-$runtime ${mode}_$arch"; |
131 } | 132 } |
132 | 133 |
133 List<String> get batchRunnerArguments() => ['-batch']; | 134 List<String> get batchRunnerArguments() => ['-batch']; |
134 List<String> get batchTestArguments() => commands.last().arguments; | 135 List<String> get batchTestArguments() => commands.last().arguments; |
135 | 136 |
136 void completed() { completedHandler(this); } | 137 void completed() { completedHandler(this); } |
137 | 138 |
138 bool get usesWebDriver() => configuration['component'] == 'webdriver'; | 139 bool get usesWebDriver() => (const ['chrome', 'ff', 'safari', 'ie', 'opera']) |
| 140 .indexOf(configuration['runtime']) >= 0; |
139 } | 141 } |
140 | 142 |
141 | 143 |
142 /** | 144 /** |
143 * BrowserTestCase has an extra compilation command that is run in a separate | 145 * BrowserTestCase has an extra compilation command that is run in a separate |
144 * process, before the regular test is run as in the base class [TestCase]. | 146 * process, before the regular test is run as in the base class [TestCase]. |
145 * If the compilation command fails, then the rest of the test is not run. | 147 * If the compilation command fails, then the rest of the test is not run. |
146 */ | 148 */ |
147 class BrowserTestCase extends TestCase { | 149 class BrowserTestCase extends TestCase { |
148 /** | 150 /** |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
222 Duration this.time) { | 224 Duration this.time) { |
223 testCase.output = this; | 225 testCase.output = this; |
224 diagnostics = []; | 226 diagnostics = []; |
225 } | 227 } |
226 | 228 |
227 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu
t, | 229 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu
t, |
228 List<String> stdout, List<String> stderr, Dur
ation time) { | 230 List<String> stdout, List<String> stderr, Dur
ation time) { |
229 if (testCase is BrowserTestCase) { | 231 if (testCase is BrowserTestCase) { |
230 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, | 232 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, |
231 stdout, stderr, time); | 233 stdout, stderr, time); |
232 } else if (testCase.configuration['component'] == 'dartc') { | 234 } else if (testCase.configuration['compiler'] == 'dartc') { |
233 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, | 235 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, |
234 stdout, stderr, time); | 236 stdout, stderr, time); |
235 } | 237 } |
236 return new TestOutputImpl(testCase, exitCode, timedOut, | 238 return new TestOutputImpl(testCase, exitCode, timedOut, |
237 stdout, stderr, time); | 239 stdout, stderr, time); |
238 } | 240 } |
239 | 241 |
240 String get result() => | 242 String get result() => |
241 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); | 243 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); |
242 | 244 |
(...skipping 284 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
527 if (currentStep == totalSteps) { // done with test command | 529 if (currentStep == totalSteps) { // done with test command |
528 testComplete(exitCode); | 530 testComplete(exitCode); |
529 } else if (exitCode != 0) { | 531 } else if (exitCode != 0) { |
530 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 532 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
531 testComplete(exitCode); | 533 testComplete(exitCode); |
532 } else { | 534 } else { |
533 stderr.add('test.dart: Compilion finished $suffix\n'); | 535 stderr.add('test.dart: Compilion finished $suffix\n'); |
534 stdout.add('test.dart: Compilion finished $suffix\n'); | 536 stdout.add('test.dart: Compilion finished $suffix\n'); |
535 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && | 537 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && |
536 !testCase.configuration['noBatch']) { | 538 !testCase.configuration['noBatch']) { |
537 // Note: processQueue will always be non-null for component == webdriver | 539 // Note: processQueue will always be non-null for runtime == ie, ff, |
538 // (It is only null for component == vm) | 540 // safari, chrome, opera. (It is only null for runtime == vm) |
539 processQueue._getBatchRunner(testCase).startTest(testCase); | 541 processQueue._getBatchRunner(testCase).startTest(testCase); |
540 } else { | 542 } else { |
541 runCommand(testCase.commands[currentStep++], stepExitHandler); | 543 runCommand(testCase.commands[currentStep++], stepExitHandler); |
542 } | 544 } |
543 } | 545 } |
544 } | 546 } |
545 | 547 |
546 Function makeReadHandler(StringInputStream source, List<String> destination) { | 548 Function makeReadHandler(StringInputStream source, List<String> destination) { |
547 return () { | 549 return () { |
548 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. | 550 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. |
(...skipping 487 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1036 void _terminateBatchRunners() { | 1038 void _terminateBatchRunners() { |
1037 for (var runners in _batchProcesses.getValues()) { | 1039 for (var runners in _batchProcesses.getValues()) { |
1038 for (var runner in runners) { | 1040 for (var runner in runners) { |
1039 runner.terminate(); | 1041 runner.terminate(); |
1040 } | 1042 } |
1041 } | 1043 } |
1042 } | 1044 } |
1043 | 1045 |
1044 BatchRunnerProcess _getBatchRunner(TestCase test) { | 1046 BatchRunnerProcess _getBatchRunner(TestCase test) { |
1045 // Start batch processes if needed | 1047 // Start batch processes if needed |
1046 var component = test.configuration['component']; | 1048 var compiler = test.configuration['compiler']; |
1047 var runners = _batchProcesses[component]; | 1049 var runners = _batchProcesses[compiler]; |
1048 if (runners == null) { | 1050 if (runners == null) { |
1049 runners = new List<BatchRunnerProcess>(_maxProcesses); | 1051 runners = new List<BatchRunnerProcess>(_maxProcesses); |
1050 for (int i = 0; i < _maxProcesses; i++) { | 1052 for (int i = 0; i < _maxProcesses; i++) { |
1051 runners[i] = new BatchRunnerProcess(test); | 1053 runners[i] = new BatchRunnerProcess(test); |
1052 } | 1054 } |
1053 _batchProcesses[component] = runners; | 1055 _batchProcesses[compiler] = runners; |
1054 } | 1056 } |
1055 | 1057 |
1056 for (var runner in runners) { | 1058 for (var runner in runners) { |
1057 if (!runner.active) return runner; | 1059 if (!runner.active) return runner; |
1058 } | 1060 } |
1059 throw new Exception('Unable to find inactive batch runner.'); | 1061 throw new Exception('Unable to find inactive batch runner.'); |
1060 } | 1062 } |
1061 | 1063 |
1062 void _tryRunTest() { | 1064 void _tryRunTest() { |
1063 _checkDone(); | 1065 _checkDone(); |
(...skipping 22 matching lines...) Expand all Loading... |
1086 } | 1088 } |
1087 _progress.start(test); | 1089 _progress.start(test); |
1088 Function oldCallback = test.completedHandler; | 1090 Function oldCallback = test.completedHandler; |
1089 Function wrapper = (TestCase test_arg) { | 1091 Function wrapper = (TestCase test_arg) { |
1090 _numProcesses--; | 1092 _numProcesses--; |
1091 _progress.done(test_arg); | 1093 _progress.done(test_arg); |
1092 _tryRunTest(); | 1094 _tryRunTest(); |
1093 oldCallback(test_arg); | 1095 oldCallback(test_arg); |
1094 }; | 1096 }; |
1095 test.completedHandler = wrapper; | 1097 test.completedHandler = wrapper; |
1096 if (test.configuration['component'] == 'dartc' && | 1098 if (test.configuration['compiler'] == 'dartc' && |
1097 test.displayName != 'dartc/junit_tests') { | 1099 test.displayName != 'dartc/junit_tests') { |
1098 _getBatchRunner(test).startTest(test); | 1100 _getBatchRunner(test).startTest(test); |
1099 } else { | 1101 } else { |
1100 // Once we've actually failed a test, technically, we wouldn't need to | 1102 // Once we've actually failed a test, technically, we wouldn't need to |
1101 // bother retrying any subsequent tests since the bot is already red. | 1103 // bother retrying any subsequent tests since the bot is already red. |
1102 // However, we continue to retry tests until we have actually failed | 1104 // However, we continue to retry tests until we have actually failed |
1103 // four tests (arbitrarily chosen) for more debugable output, so that | 1105 // four tests (arbitrarily chosen) for more debugable output, so that |
1104 // the developer doesn't waste his or her time trying to fix a bunch of | 1106 // the developer doesn't waste his or her time trying to fix a bunch of |
1105 // tests that appear to be broken but were actually just flakes that | 1107 // tests that appear to be broken but were actually just flakes that |
1106 // didn't get retried because there had already been one failure. | 1108 // didn't get retried because there had already been one failure. |
1107 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1109 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
1108 new RunningProcess(test, allowRetry, this).start(); | 1110 new RunningProcess(test, allowRetry, this).start(); |
1109 } | 1111 } |
1110 _numProcesses++; | 1112 _numProcesses++; |
1111 } | 1113 } |
1112 } | 1114 } |
1113 } | 1115 } |
OLD | NEW |