| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 118 final newCommand = new Command(newExecutablePath, newArguments); | 118 final newCommand = new Command(newExecutablePath, newArguments); |
| 119 newCommands.add(newCommand); | 119 newCommands.add(newCommand); |
| 120 // If there are extra spaces inside the prefix or suffix, this fails. | 120 // If there are extra spaces inside the prefix or suffix, this fails. |
| 121 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), | 121 Expect.stringEquals('$prefix ${c.commandLine} $suffix'.trim(), |
| 122 newCommand.commandLine); | 122 newCommand.commandLine); |
| 123 } | 123 } |
| 124 commands = newCommands; | 124 commands = newCommands; |
| 125 } | 125 } |
| 126 } | 126 } |
| 127 | 127 |
| 128 int get timeout() { | 128 int get timeout { |
| 129 if (expectedOutcomes.contains(SLOW)) { | 129 if (expectedOutcomes.contains(SLOW)) { |
| 130 return configuration['timeout'] * SLOW_TIMEOUT_MULTIPLIER; | 130 return configuration['timeout'] * SLOW_TIMEOUT_MULTIPLIER; |
| 131 } else { | 131 } else { |
| 132 return configuration['timeout']; | 132 return configuration['timeout']; |
| 133 } | 133 } |
| 134 } | 134 } |
| 135 | 135 |
| 136 String get configurationString() { | 136 String get configurationString { |
| 137 final compiler = configuration['compiler']; | 137 final compiler = configuration['compiler']; |
| 138 final runtime = configuration['runtime']; | 138 final runtime = configuration['runtime']; |
| 139 final mode = configuration['mode']; | 139 final mode = configuration['mode']; |
| 140 final arch = configuration['arch']; | 140 final arch = configuration['arch']; |
| 141 return "$compiler-$runtime ${mode}_$arch"; | 141 return "$compiler-$runtime ${mode}_$arch"; |
| 142 } | 142 } |
| 143 | 143 |
| 144 List<String> get batchRunnerArguments() => ['-batch']; | 144 List<String> get batchRunnerArguments => ['-batch']; |
| 145 List<String> get batchTestArguments() => commands.last().arguments; | 145 List<String> get batchTestArguments => commands.last().arguments; |
| 146 | 146 |
| 147 void completed() { completedHandler(this); } | 147 void completed() { completedHandler(this); } |
| 148 | 148 |
| 149 bool get usesWebDriver() => Contains( | 149 bool get usesWebDriver => Contains( |
| 150 configuration['runtime'], | 150 configuration['runtime'], |
| 151 const ['chrome', 'dartium', 'ff', 'safari', 'ie', 'opera']); | 151 const ['chrome', 'dartium', 'ff', 'safari', 'ie', 'opera']); |
| 152 } | 152 } |
| 153 | 153 |
| 154 | 154 |
| 155 /** | 155 /** |
| 156 * BrowserTestCase has an extra compilation command that is run in a separate | 156 * BrowserTestCase has an extra compilation command that is run in a separate |
| 157 * process, before the regular test is run as in the base class [TestCase]. | 157 * process, before the regular test is run as in the base class [TestCase]. |
| 158 * If the compilation command fails, then the rest of the test is not run. | 158 * If the compilation command fails, then the rest of the test is not run. |
| 159 */ | 159 */ |
| 160 class BrowserTestCase extends TestCase { | 160 class BrowserTestCase extends TestCase { |
| 161 /** | 161 /** |
| 162 * Indicates the number of potential retries remaining, to compensate for | 162 * Indicates the number of potential retries remaining, to compensate for |
| 163 * flaky browser tests. | 163 * flaky browser tests. |
| 164 */ | 164 */ |
| 165 int numRetries; | 165 int numRetries; |
| 166 | 166 |
| 167 BrowserTestCase(displayName, commands, configuration, completedHandler, | 167 BrowserTestCase(displayName, commands, configuration, completedHandler, |
| 168 expectedOutcomes, [isNegative = false]) | 168 expectedOutcomes, [isNegative = false]) |
| 169 : super(displayName, commands, configuration, completedHandler, | 169 : super(displayName, commands, configuration, completedHandler, |
| 170 expectedOutcomes, isNegative) { | 170 expectedOutcomes, isNegative) { |
| 171 numRetries = 2; // Allow two retries to compensate for flaky browser tests. | 171 numRetries = 2; // Allow two retries to compensate for flaky browser tests. |
| 172 } | 172 } |
| 173 | 173 |
| 174 List<String> get _lastArguments() => commands.last().arguments; | 174 List<String> get _lastArguments => commands.last().arguments; |
| 175 | 175 |
| 176 List<String> get batchRunnerArguments() => [_lastArguments[0], '--batch']; | 176 List<String> get batchRunnerArguments => [_lastArguments[0], '--batch']; |
| 177 | 177 |
| 178 List<String> get batchTestArguments() => | 178 List<String> get batchTestArguments => |
| 179 _lastArguments.getRange(1, _lastArguments.length - 1); | 179 _lastArguments.getRange(1, _lastArguments.length - 1); |
| 180 } | 180 } |
| 181 | 181 |
| 182 | 182 |
| 183 /** | 183 /** |
| 184 * TestOutput records the output of a completed test: the process's exit code, | 184 * TestOutput records the output of a completed test: the process's exit code, |
| 185 * the standard output and standard error, whether the process timed out, and | 185 * the standard output and standard error, whether the process timed out, and |
| 186 * the time the process took to run. It also contains a pointer to the | 186 * the time the process took to run. It also contains a pointer to the |
| 187 * [TestCase] this is the output of. | 187 * [TestCase] this is the output of. |
| 188 */ | 188 */ |
| 189 interface TestOutput default TestOutputImpl { | 189 interface TestOutput default TestOutputImpl { |
| 190 TestOutput.fromCase(TestCase testCase, int exitCode, bool timedOut, | 190 TestOutput.fromCase(TestCase testCase, int exitCode, bool timedOut, |
| 191 List<String> stdout, List<String> stderr, Duration time); | 191 List<String> stdout, List<String> stderr, Duration time); |
| 192 | 192 |
| 193 String get result(); | 193 String get result; |
| 194 | 194 |
| 195 bool get unexpectedOutput(); | 195 bool get unexpectedOutput; |
| 196 | 196 |
| 197 bool get hasCrashed(); | 197 bool get hasCrashed; |
| 198 | 198 |
| 199 bool get hasTimedOut(); | 199 bool get hasTimedOut; |
| 200 | 200 |
| 201 bool get didFail(); | 201 bool get didFail; |
| 202 | 202 |
| 203 bool requestRetry; | 203 bool requestRetry; |
| 204 | 204 |
| 205 Duration get time(); | 205 Duration get time; |
| 206 | 206 |
| 207 int get exitCode(); | 207 int get exitCode; |
| 208 | 208 |
| 209 List<String> get stdout(); | 209 List<String> get stdout; |
| 210 | 210 |
| 211 List<String> get stderr(); | 211 List<String> get stderr; |
| 212 | 212 |
| 213 List<String> get diagnostics(); | 213 List<String> get diagnostics; |
| 214 } | 214 } |
| 215 | 215 |
| 216 class TestOutputImpl implements TestOutput { | 216 class TestOutputImpl implements TestOutput { |
| 217 TestCase testCase; | 217 TestCase testCase; |
| 218 int exitCode; | 218 int exitCode; |
| 219 bool timedOut; | 219 bool timedOut; |
| 220 bool failed = false; | 220 bool failed = false; |
| 221 List<String> stdout; | 221 List<String> stdout; |
| 222 List<String> stderr; | 222 List<String> stderr; |
| 223 Duration time; | 223 Duration time; |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 257 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, | 257 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, |
| 258 stdout, stderr, time); | 258 stdout, stderr, time); |
| 259 } else if (testCase.configuration['compiler'] == 'dartc') { | 259 } else if (testCase.configuration['compiler'] == 'dartc') { |
| 260 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, | 260 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, |
| 261 stdout, stderr, time); | 261 stdout, stderr, time); |
| 262 } | 262 } |
| 263 return new TestOutputImpl(testCase, exitCode, timedOut, | 263 return new TestOutputImpl(testCase, exitCode, timedOut, |
| 264 stdout, stderr, time); | 264 stdout, stderr, time); |
| 265 } | 265 } |
| 266 | 266 |
| 267 String get result() => | 267 String get result => |
| 268 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); | 268 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); |
| 269 | 269 |
| 270 bool get unexpectedOutput() => !testCase.expectedOutcomes.contains(result); | 270 bool get unexpectedOutput => !testCase.expectedOutcomes.contains(result); |
| 271 | 271 |
| 272 bool get hasCrashed() { | 272 bool get hasCrashed { |
| 273 // The Java dartc runner and dart2js exits with code 253 in case | 273 // The Java dartc runner and dart2js exits with code 253 in case |
| 274 // of unhandled exceptions. | 274 // of unhandled exceptions. |
| 275 if (exitCode == 253) return true; | 275 if (exitCode == 253) return true; |
| 276 if (Platform.operatingSystem == 'windows') { | 276 if (Platform.operatingSystem == 'windows') { |
| 277 // The VM uses std::abort to terminate on asserts. | 277 // The VM uses std::abort to terminate on asserts. |
| 278 // std::abort terminates with exit code 3 on Windows. | 278 // std::abort terminates with exit code 3 on Windows. |
| 279 if (exitCode == 3) { | 279 if (exitCode == 3) { |
| 280 return !timedOut; | 280 return !timedOut; |
| 281 } | 281 } |
| 282 return (!timedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0)); | 282 return (!timedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0)); |
| 283 } | 283 } |
| 284 return !timedOut && ((exitCode < 0)); | 284 return !timedOut && ((exitCode < 0)); |
| 285 } | 285 } |
| 286 | 286 |
| 287 bool get hasTimedOut() => timedOut; | 287 bool get hasTimedOut => timedOut; |
| 288 | 288 |
| 289 bool get didFail() { | 289 bool get didFail { |
| 290 return (exitCode != 0 && !hasCrashed); | 290 return (exitCode != 0 && !hasCrashed); |
| 291 } | 291 } |
| 292 | 292 |
| 293 // Reverse result of a negative test. | 293 // Reverse result of a negative test. |
| 294 bool get hasFailed() => testCase.isNegative ? !didFail : didFail; | 294 bool get hasFailed => testCase.isNegative ? !didFail : didFail; |
| 295 | 295 |
| 296 } | 296 } |
| 297 | 297 |
| 298 class BrowserTestOutputImpl extends TestOutputImpl { | 298 class BrowserTestOutputImpl extends TestOutputImpl { |
| 299 BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : | 299 BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : |
| 300 super(testCase, exitCode, timedOut, stdout, stderr, time); | 300 super(testCase, exitCode, timedOut, stdout, stderr, time); |
| 301 | 301 |
| 302 bool get didFail() { | 302 bool get didFail { |
| 303 // Browser case: | 303 // Browser case: |
| 304 // If the browser test failed, it may have been because DumpRenderTree | 304 // If the browser test failed, it may have been because DumpRenderTree |
| 305 // and the virtual framebuffer X server didn't hook up, or DRT crashed with | 305 // and the virtual framebuffer X server didn't hook up, or DRT crashed with |
| 306 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, | 306 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, |
| 307 // so we have to do this check first. | 307 // so we have to do this check first. |
| 308 for (String line in super.stderr) { | 308 for (String line in super.stderr) { |
| 309 if (line.contains('Gtk-WARNING **: cannot open display: :99') || | 309 if (line.contains('Gtk-WARNING **: cannot open display: :99') || |
| 310 line.contains('Failed to run command. return code=1')) { | 310 line.contains('Failed to run command. return code=1')) { |
| 311 // If we get the X server error, or DRT crashes with a core dump, retry | 311 // If we get the X server error, or DRT crashes with a core dump, retry |
| 312 // the test. | 312 // the test. |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 345 final int ERROR_LEVEL = 0; | 345 final int ERROR_LEVEL = 0; |
| 346 final int ERROR_TYPE = 1; | 346 final int ERROR_TYPE = 1; |
| 347 final int FORMATTED_ERROR = 7; | 347 final int FORMATTED_ERROR = 7; |
| 348 | 348 |
| 349 bool alreadyComputed = false; | 349 bool alreadyComputed = false; |
| 350 bool failResult; | 350 bool failResult; |
| 351 AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : | 351 AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : |
| 352 super(testCase, exitCode, timedOut, stdout, stderr, time) { | 352 super(testCase, exitCode, timedOut, stdout, stderr, time) { |
| 353 } | 353 } |
| 354 | 354 |
| 355 bool get didFail() { | 355 bool get didFail { |
| 356 if (!alreadyComputed) { | 356 if (!alreadyComputed) { |
| 357 failResult = _didFail(); | 357 failResult = _didFail(); |
| 358 alreadyComputed = true; | 358 alreadyComputed = true; |
| 359 } | 359 } |
| 360 return failResult; | 360 return failResult; |
| 361 } | 361 } |
| 362 | 362 |
| 363 bool _didFail() { | 363 bool _didFail() { |
| 364 if (hasCrashed) return false; | 364 if (hasCrashed) return false; |
| 365 | 365 |
| (...skipping 309 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 675 Timer _timer; | 675 Timer _timer; |
| 676 | 676 |
| 677 bool _isWebDriver; | 677 bool _isWebDriver; |
| 678 | 678 |
| 679 BatchRunnerProcess(TestCase testCase) { | 679 BatchRunnerProcess(TestCase testCase) { |
| 680 _executable = testCase.commands.last().executable; | 680 _executable = testCase.commands.last().executable; |
| 681 _batchArguments = testCase.batchRunnerArguments; | 681 _batchArguments = testCase.batchRunnerArguments; |
| 682 _isWebDriver = testCase.usesWebDriver; | 682 _isWebDriver = testCase.usesWebDriver; |
| 683 } | 683 } |
| 684 | 684 |
| 685 bool get active() => _currentTest != null; | 685 bool get active => _currentTest != null; |
| 686 | 686 |
| 687 void startTest(TestCase testCase) { | 687 void startTest(TestCase testCase) { |
| 688 Expect.isNull(_currentTest); | 688 Expect.isNull(_currentTest); |
| 689 _currentTest = testCase; | 689 _currentTest = testCase; |
| 690 if (_process === null) { | 690 if (_process === null) { |
| 691 // Start process if not yet started. | 691 // Start process if not yet started. |
| 692 _executable = testCase.commands.last().executable; | 692 _executable = testCase.commands.last().executable; |
| 693 _startProcess(() { | 693 _startProcess(() { |
| 694 doStartTest(testCase); | 694 doStartTest(testCase); |
| 695 }); | 695 }); |
| (...skipping 294 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 990 _terminateBatchRunners(); | 990 _terminateBatchRunners(); |
| 991 _cleanupAndMarkDone(); | 991 _cleanupAndMarkDone(); |
| 992 } | 992 } |
| 993 } | 993 } |
| 994 } | 994 } |
| 995 | 995 |
| 996 /** | 996 /** |
| 997 * True if we are using a browser + platform combination that needs the | 997 * True if we are using a browser + platform combination that needs the |
| 998 * Selenium server jar. | 998 * Selenium server jar. |
| 999 */ | 999 */ |
| 1000 bool get _needsSelenium() => Platform.operatingSystem == 'macos' && | 1000 bool get _needsSelenium => Platform.operatingSystem == 'macos' && |
| 1001 browserUsed == 'safari'; | 1001 browserUsed == 'safari'; |
| 1002 | 1002 |
| 1003 /** True if the Selenium Server is ready to be used. */ | 1003 /** True if the Selenium Server is ready to be used. */ |
| 1004 bool get _isSeleniumAvailable() => _seleniumServer != null || | 1004 bool get _isSeleniumAvailable => _seleniumServer != null || |
| 1005 _seleniumAlreadyRunning; | 1005 _seleniumAlreadyRunning; |
| 1006 | 1006 |
| 1007 /** | 1007 /** |
| 1008 * Restart all the processes that have been waiting/stopped for the server to | 1008 * Restart all the processes that have been waiting/stopped for the server to |
| 1009 * start up. If we just call this once we end up with a single-"threaded" run. | 1009 * start up. If we just call this once we end up with a single-"threaded" run. |
| 1010 */ | 1010 */ |
| 1011 void resumeTesting() { | 1011 void resumeTesting() { |
| 1012 for (int i = 0; i < _maxProcesses; i++) _tryRunTest(); | 1012 for (int i = 0; i < _maxProcesses; i++) _tryRunTest(); |
| 1013 } | 1013 } |
| 1014 | 1014 |
| (...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1186 // the developer doesn't waste his or her time trying to fix a bunch of | 1186 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1187 // tests that appear to be broken but were actually just flakes that | 1187 // tests that appear to be broken but were actually just flakes that |
| 1188 // didn't get retried because there had already been one failure. | 1188 // didn't get retried because there had already been one failure. |
| 1189 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1189 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1190 new RunningProcess(test, allowRetry, this).start(); | 1190 new RunningProcess(test, allowRetry, this).start(); |
| 1191 } | 1191 } |
| 1192 _numProcesses++; | 1192 _numProcesses++; |
| 1193 } | 1193 } |
| 1194 } | 1194 } |
| 1195 } | 1195 } |
| OLD | NEW |