OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
179 | 179 |
180 String get result(); | 180 String get result(); |
181 | 181 |
182 bool get unexpectedOutput(); | 182 bool get unexpectedOutput(); |
183 | 183 |
184 bool get hasCrashed(); | 184 bool get hasCrashed(); |
185 | 185 |
186 bool get hasTimedOut(); | 186 bool get hasTimedOut(); |
187 | 187 |
188 bool get didFail(); | 188 bool get didFail(); |
189 | 189 |
190 Duration get time(); | 190 Duration get time(); |
191 | 191 |
192 List<String> get stdout(); | 192 List<String> get stdout(); |
193 | 193 |
194 List<String> get stderr(); | 194 List<String> get stderr(); |
195 | 195 |
196 List<String> get diagnostics(); | 196 List<String> get diagnostics(); |
197 } | 197 } |
198 | 198 |
199 class TestOutputImpl implements TestOutput { | 199 class TestOutputImpl implements TestOutput { |
200 TestCase testCase; | 200 TestCase testCase; |
201 int exitCode; | 201 int exitCode; |
202 bool timedOut; | 202 bool timedOut; |
203 bool failed = false; | 203 bool failed = false; |
204 List<String> stdout; | 204 List<String> stdout; |
205 List<String> stderr; | 205 List<String> stderr; |
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
346 return _didMultitestFail(errors, staticWarnings); | 346 return _didMultitestFail(errors, staticWarnings); |
347 } | 347 } |
348 return _didStandardTestFail(errors, staticWarnings); | 348 return _didStandardTestFail(errors, staticWarnings); |
349 } | 349 } |
350 | 350 |
351 bool _didMultitestFail(List errors, List staticWarnings) { | 351 bool _didMultitestFail(List errors, List staticWarnings) { |
352 Set<String> outcome = testCase.info.multitestOutcome; | 352 Set<String> outcome = testCase.info.multitestOutcome; |
353 Expect.isNotNull(outcome); | 353 Expect.isNotNull(outcome); |
354 if (outcome.contains('compile-time error') && errors.length > 0) { | 354 if (outcome.contains('compile-time error') && errors.length > 0) { |
355 return true; | 355 return true; |
356 } else if (outcome.contains('static type warning') | 356 } else if (outcome.contains('static type warning') |
357 && staticWarnings.length > 0) { | 357 && staticWarnings.length > 0) { |
358 return true; | 358 return true; |
359 } else if (outcome.isEmpty() | 359 } else if (outcome.isEmpty() |
360 && (errors.length > 0 || staticWarnings.length > 0)) { | 360 && (errors.length > 0 || staticWarnings.length > 0)) { |
361 return true; | 361 return true; |
362 } | 362 } |
363 return false; | 363 return false; |
364 } | 364 } |
365 | 365 |
366 bool _didStandardTestFail(List errors, List staticWarnings) { | 366 bool _didStandardTestFail(List errors, List staticWarnings) { |
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
491 */ | 491 */ |
492 void testComplete(int exitCode) { | 492 void testComplete(int exitCode) { |
493 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, | 493 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, |
494 stderr, new Date.now().difference(startTime)); | 494 stderr, new Date.now().difference(startTime)); |
495 timeoutTimer.cancel(); | 495 timeoutTimer.cancel(); |
496 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { | 496 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { |
497 print(testCase.displayName); | 497 print(testCase.displayName); |
498 for (var line in testCase.output.stderr) print(line); | 498 for (var line in testCase.output.stderr) print(line); |
499 for (var line in testCase.output.stdout) print(line); | 499 for (var line in testCase.output.stdout) print(line); |
500 } | 500 } |
501 if (allowRetries && testCase.usesWebDriver | 501 if (allowRetries && testCase.usesWebDriver |
502 && testCase.output.unexpectedOutput && testCase.numRetries > 0) { | 502 && testCase.output.unexpectedOutput && testCase.numRetries > 0) { |
503 // Selenium tests can be flaky. Try rerunning. | 503 // Selenium tests can be flaky. Try rerunning. |
504 testCase.output.requestRetry = true; | 504 testCase.output.requestRetry = true; |
505 } | 505 } |
506 if (testCase.output.requestRetry) { | 506 if (testCase.output.requestRetry) { |
507 testCase.output.requestRetry = false; | 507 testCase.output.requestRetry = false; |
508 this.timedOut = false; | 508 this.timedOut = false; |
509 testCase.dynamic.numRetries--; | 509 testCase.dynamic.numRetries--; |
510 print("Potential flake. Re-running ${testCase.displayName} " + | 510 print("Potential flake. Re-running ${testCase.displayName} " + |
511 "(${testCase.dynamic.numRetries} attempt(s) remains)"); | 511 "(${testCase.dynamic.numRetries} attempt(s) remains)"); |
(...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
833 bool _startingServer = false; | 833 bool _startingServer = false; |
834 | 834 |
835 /** True if we find that there is already a selenium jar running. */ | 835 /** True if we find that there is already a selenium jar running. */ |
836 bool _seleniumAlreadyRunning = false; | 836 bool _seleniumAlreadyRunning = false; |
837 | 837 |
838 ProcessQueue(int this._maxProcesses, | 838 ProcessQueue(int this._maxProcesses, |
839 String progress, | 839 String progress, |
840 Date startTime, | 840 Date startTime, |
841 bool printTiming, | 841 bool printTiming, |
842 Function this._enqueueMoreWork, | 842 Function this._enqueueMoreWork, |
843 [bool verbose = false, | 843 [bool this._verbose = false, |
844 bool listTests = false, | 844 bool this._listTests = false, |
845 bool keepGeneratedTests = false]) | 845 bool this._keepGeneratedTests = false]) |
846 : _verbose = verbose, | 846 : _tests = new Queue<TestCase>(), |
847 _listTests = listTests, | |
848 _keepGeneratedTests = keepGeneratedTests, | |
849 _tests = new Queue<TestCase>(), | |
850 _progress = new ProgressIndicator.fromName(progress, | 847 _progress = new ProgressIndicator.fromName(progress, |
851 startTime, | 848 startTime, |
852 printTiming), | 849 printTiming), |
853 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), | 850 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), |
854 _testCache = new Map<String, List<TestInformation>>() { | 851 _testCache = new Map<String, List<TestInformation>>() { |
855 if (!_enqueueMoreWork(this)) _progress.allDone(); | 852 if (!_enqueueMoreWork(this)) _progress.allDone(); |
856 } | 853 } |
857 | 854 |
858 /** | 855 /** |
859 * Registers a TestSuite so that all of its tests will be run. | 856 * Registers a TestSuite so that all of its tests will be run. |
(...skipping 251 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1111 // the developer doesn't waste his or her time trying to fix a bunch of | 1108 // the developer doesn't waste his or her time trying to fix a bunch of |
1112 // tests that appear to be broken but were actually just flakes that | 1109 // tests that appear to be broken but were actually just flakes that |
1113 // didn't get retried because there had already been one failure. | 1110 // didn't get retried because there had already been one failure. |
1114 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1111 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
1115 new RunningProcess(test, allowRetry, this).start(); | 1112 new RunningProcess(test, allowRetry, this).start(); |
1116 } | 1113 } |
1117 _numProcesses++; | 1114 _numProcesses++; |
1118 } | 1115 } |
1119 } | 1116 } |
1120 } | 1117 } |
OLD | NEW |