OLD | NEW |
---|---|
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), | 117 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), |
118 newCommand.commandLine); | 118 newCommand.commandLine); |
119 } | 119 } |
120 commands = newCommands; | 120 commands = newCommands; |
121 } | 121 } |
122 } | 122 } |
123 | 123 |
124 int get timeout() => configuration['timeout']; | 124 int get timeout() => configuration['timeout']; |
125 | 125 |
126 String get configurationString() { | 126 String get configurationString() { |
127 final component = configuration['component']; | 127 final compiler = configuration['compiler']; |
128 final runtime = configuration['runtime']; | |
128 final mode = configuration['mode']; | 129 final mode = configuration['mode']; |
129 final arch = configuration['arch']; | 130 final arch = configuration['arch']; |
130 return "$component ${mode}_$arch"; | 131 return "$compiler-$runtime ${mode}_$arch"; |
131 } | 132 } |
132 | 133 |
133 List<String> get batchRunnerArguments() => ['-batch']; | 134 List<String> get batchRunnerArguments() => ['-batch']; |
134 List<String> get batchTestArguments() => commands.last().arguments; | 135 List<String> get batchTestArguments() => commands.last().arguments; |
135 | 136 |
136 void completed() { completedHandler(this); } | 137 void completed() { completedHandler(this); } |
137 | 138 |
138 bool get usesWebDriver() => configuration['component'] == 'webdriver'; | 139 bool get usesWebDriver() => configuration['runtime'] == 'chrome' || |
140 configuration['runtime'] == 'ff' || configuration['runtime'] == 'safari' | |
141 || configuration['runtime'] == 'ie' | |
142 || configuration['runtime'] == 'opera'; | |
Siggi Cherem (dart-lang)
2012/03/23 23:57:33
an alternative suggestion:
bool get usesWebDriver
ahe
2012/03/24 13:56:40
+1
Emily Fortuna
2012/03/26 20:52:32
Done.
| |
139 } | 143 } |
140 | 144 |
141 | 145 |
142 /** | 146 /** |
143 * BrowserTestCase has an extra compilation command that is run in a separate | 147 * BrowserTestCase has an extra compilation command that is run in a separate |
144 * process, before the regular test is run as in the base class [TestCase]. | 148 * process, before the regular test is run as in the base class [TestCase]. |
145 * If the compilation command fails, then the rest of the test is not run. | 149 * If the compilation command fails, then the rest of the test is not run. |
146 */ | 150 */ |
147 class BrowserTestCase extends TestCase { | 151 class BrowserTestCase extends TestCase { |
148 /** | 152 /** |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
222 Duration this.time) { | 226 Duration this.time) { |
223 testCase.output = this; | 227 testCase.output = this; |
224 diagnostics = []; | 228 diagnostics = []; |
225 } | 229 } |
226 | 230 |
227 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu t, | 231 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu t, |
228 List<String> stdout, List<String> stderr, Dur ation time) { | 232 List<String> stdout, List<String> stderr, Dur ation time) { |
229 if (testCase is BrowserTestCase) { | 233 if (testCase is BrowserTestCase) { |
230 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, | 234 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, |
231 stdout, stderr, time); | 235 stdout, stderr, time); |
232 } else if (testCase.configuration['component'] == 'dartc') { | 236 } else if (testCase.configuration['compiler'] == 'dartc') { |
233 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, | 237 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, |
234 stdout, stderr, time); | 238 stdout, stderr, time); |
235 } | 239 } |
236 return new TestOutputImpl(testCase, exitCode, timedOut, | 240 return new TestOutputImpl(testCase, exitCode, timedOut, |
237 stdout, stderr, time); | 241 stdout, stderr, time); |
238 } | 242 } |
239 | 243 |
240 String get result() => | 244 String get result() => |
241 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); | 245 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); |
242 | 246 |
(...skipping 284 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
527 if (currentStep == totalSteps) { // done with test command | 531 if (currentStep == totalSteps) { // done with test command |
528 testComplete(exitCode); | 532 testComplete(exitCode); |
529 } else if (exitCode != 0) { | 533 } else if (exitCode != 0) { |
530 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 534 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
531 testComplete(exitCode); | 535 testComplete(exitCode); |
532 } else { | 536 } else { |
533 stderr.add('test.dart: Compilion finished $suffix\n'); | 537 stderr.add('test.dart: Compilion finished $suffix\n'); |
534 stdout.add('test.dart: Compilion finished $suffix\n'); | 538 stdout.add('test.dart: Compilion finished $suffix\n'); |
535 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && | 539 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && |
536 !testCase.configuration['noBatch']) { | 540 !testCase.configuration['noBatch']) { |
537 // Note: processQueue will always be non-null for component == webdriver | 541 // Note: processQueue will always be non-null for runtime == ie, ff, |
538 // (It is only null for component == vm) | 542 // safari, chrome, opera. (It is only null for runtime == vm) |
539 processQueue._getBatchRunner(testCase).startTest(testCase); | 543 processQueue._getBatchRunner(testCase).startTest(testCase); |
540 } else { | 544 } else { |
541 runCommand(testCase.commands[currentStep++], stepExitHandler); | 545 runCommand(testCase.commands[currentStep++], stepExitHandler); |
542 } | 546 } |
543 } | 547 } |
544 } | 548 } |
545 | 549 |
546 Function makeReadHandler(StringInputStream source, List<String> destination) { | 550 Function makeReadHandler(StringInputStream source, List<String> destination) { |
547 return () { | 551 return () { |
548 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. | 552 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. |
(...skipping 487 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1036 void _terminateBatchRunners() { | 1040 void _terminateBatchRunners() { |
1037 for (var runners in _batchProcesses.getValues()) { | 1041 for (var runners in _batchProcesses.getValues()) { |
1038 for (var runner in runners) { | 1042 for (var runner in runners) { |
1039 runner.terminate(); | 1043 runner.terminate(); |
1040 } | 1044 } |
1041 } | 1045 } |
1042 } | 1046 } |
1043 | 1047 |
1044 BatchRunnerProcess _getBatchRunner(TestCase test) { | 1048 BatchRunnerProcess _getBatchRunner(TestCase test) { |
1045 // Start batch processes if needed | 1049 // Start batch processes if needed |
1046 var component = test.configuration['component']; | 1050 var compiler = test.configuration['compiler']; |
1047 var runners = _batchProcesses[component]; | 1051 var runners = _batchProcesses[compiler]; |
1048 if (runners == null) { | 1052 if (runners == null) { |
1049 runners = new List<BatchRunnerProcess>(_maxProcesses); | 1053 runners = new List<BatchRunnerProcess>(_maxProcesses); |
1050 for (int i = 0; i < _maxProcesses; i++) { | 1054 for (int i = 0; i < _maxProcesses; i++) { |
1051 runners[i] = new BatchRunnerProcess(test); | 1055 runners[i] = new BatchRunnerProcess(test); |
1052 } | 1056 } |
1053 _batchProcesses[component] = runners; | 1057 _batchProcesses[compiler] = runners; |
1054 } | 1058 } |
1055 | 1059 |
1056 for (var runner in runners) { | 1060 for (var runner in runners) { |
1057 if (!runner.active) return runner; | 1061 if (!runner.active) return runner; |
1058 } | 1062 } |
1059 throw new Exception('Unable to find inactive batch runner.'); | 1063 throw new Exception('Unable to find inactive batch runner.'); |
1060 } | 1064 } |
1061 | 1065 |
1062 void _tryRunTest() { | 1066 void _tryRunTest() { |
1063 _checkDone(); | 1067 _checkDone(); |
(...skipping 22 matching lines...) Expand all Loading... | |
1086 } | 1090 } |
1087 _progress.start(test); | 1091 _progress.start(test); |
1088 Function oldCallback = test.completedHandler; | 1092 Function oldCallback = test.completedHandler; |
1089 Function wrapper = (TestCase test_arg) { | 1093 Function wrapper = (TestCase test_arg) { |
1090 _numProcesses--; | 1094 _numProcesses--; |
1091 _progress.done(test_arg); | 1095 _progress.done(test_arg); |
1092 _tryRunTest(); | 1096 _tryRunTest(); |
1093 oldCallback(test_arg); | 1097 oldCallback(test_arg); |
1094 }; | 1098 }; |
1095 test.completedHandler = wrapper; | 1099 test.completedHandler = wrapper; |
1096 if (test.configuration['component'] == 'dartc' && | 1100 if (test.configuration['compiler'] == 'dartc' && |
1097 test.displayName != 'dartc/junit_tests') { | 1101 test.displayName != 'dartc/junit_tests') { |
1098 _getBatchRunner(test).startTest(test); | 1102 _getBatchRunner(test).startTest(test); |
1099 } else { | 1103 } else { |
1100 // Once we've actually failed a test, technically, we wouldn't need to | 1104 // Once we've actually failed a test, technically, we wouldn't need to |
1101 // bother retrying any subsequent tests since the bot is already red. | 1105 // bother retrying any subsequent tests since the bot is already red. |
1102 // However, we continue to retry tests until we have actually failed | 1106 // However, we continue to retry tests until we have actually failed |
1103 // four tests (arbitrarily chosen) for more debugable output, so that | 1107 // four tests (arbitrarily chosen) for more debugable output, so that |
1104 // the developer doesn't waste his or her time trying to fix a bunch of | 1108 // the developer doesn't waste his or her time trying to fix a bunch of |
1105 // tests that appear to be broken but were actually just flakes that | 1109 // tests that appear to be broken but were actually just flakes that |
1106 // didn't get retried because there had already been one failure. | 1110 // didn't get retried because there had already been one failure. |
1107 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1111 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
1108 new RunningProcess(test, allowRetry, this).start(); | 1112 new RunningProcess(test, allowRetry, this).start(); |
1109 } | 1113 } |
1110 _numProcesses++; | 1114 _numProcesses++; |
1111 } | 1115 } |
1112 } | 1116 } |
1113 } | 1117 } |
OLD | NEW |