| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 179 | 179 |
| 180 String get result(); | 180 String get result(); |
| 181 | 181 |
| 182 bool get unexpectedOutput(); | 182 bool get unexpectedOutput(); |
| 183 | 183 |
| 184 bool get hasCrashed(); | 184 bool get hasCrashed(); |
| 185 | 185 |
| 186 bool get hasTimedOut(); | 186 bool get hasTimedOut(); |
| 187 | 187 |
| 188 bool get didFail(); | 188 bool get didFail(); |
| 189 | 189 |
| 190 Duration get time(); | 190 Duration get time(); |
| 191 | 191 |
| 192 List<String> get stdout(); | 192 List<String> get stdout(); |
| 193 | 193 |
| 194 List<String> get stderr(); | 194 List<String> get stderr(); |
| 195 | 195 |
| 196 List<String> get diagnostics(); | 196 List<String> get diagnostics(); |
| 197 } | 197 } |
| 198 | 198 |
| 199 class TestOutputImpl implements TestOutput { | 199 class TestOutputImpl implements TestOutput { |
| 200 TestCase testCase; | 200 TestCase testCase; |
| 201 int exitCode; | 201 int exitCode; |
| 202 bool timedOut; | 202 bool timedOut; |
| 203 bool failed = false; | 203 bool failed = false; |
| 204 List<String> stdout; | 204 List<String> stdout; |
| 205 List<String> stderr; | 205 List<String> stderr; |
| (...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 346 return _didMultitestFail(errors, staticWarnings); | 346 return _didMultitestFail(errors, staticWarnings); |
| 347 } | 347 } |
| 348 return _didStandardTestFail(errors, staticWarnings); | 348 return _didStandardTestFail(errors, staticWarnings); |
| 349 } | 349 } |
| 350 | 350 |
| 351 bool _didMultitestFail(List errors, List staticWarnings) { | 351 bool _didMultitestFail(List errors, List staticWarnings) { |
| 352 Set<String> outcome = testCase.info.multitestOutcome; | 352 Set<String> outcome = testCase.info.multitestOutcome; |
| 353 Expect.isNotNull(outcome); | 353 Expect.isNotNull(outcome); |
| 354 if (outcome.contains('compile-time error') && errors.length > 0) { | 354 if (outcome.contains('compile-time error') && errors.length > 0) { |
| 355 return true; | 355 return true; |
| 356 } else if (outcome.contains('static type warning') | 356 } else if (outcome.contains('static type warning') |
| 357 && staticWarnings.length > 0) { | 357 && staticWarnings.length > 0) { |
| 358 return true; | 358 return true; |
| 359 } else if (outcome.isEmpty() | 359 } else if (outcome.isEmpty() |
| 360 && (errors.length > 0 || staticWarnings.length > 0)) { | 360 && (errors.length > 0 || staticWarnings.length > 0)) { |
| 361 return true; | 361 return true; |
| 362 } | 362 } |
| 363 return false; | 363 return false; |
| 364 } | 364 } |
| 365 | 365 |
| 366 bool _didStandardTestFail(List errors, List staticWarnings) { | 366 bool _didStandardTestFail(List errors, List staticWarnings) { |
| (...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 491 */ | 491 */ |
| 492 void testComplete(int exitCode) { | 492 void testComplete(int exitCode) { |
| 493 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, | 493 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, |
| 494 stderr, new Date.now().difference(startTime)); | 494 stderr, new Date.now().difference(startTime)); |
| 495 timeoutTimer.cancel(); | 495 timeoutTimer.cancel(); |
| 496 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { | 496 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { |
| 497 print(testCase.displayName); | 497 print(testCase.displayName); |
| 498 for (var line in testCase.output.stderr) print(line); | 498 for (var line in testCase.output.stderr) print(line); |
| 499 for (var line in testCase.output.stdout) print(line); | 499 for (var line in testCase.output.stdout) print(line); |
| 500 } | 500 } |
| 501 if (allowRetries && testCase.usesWebDriver | 501 if (allowRetries && testCase.usesWebDriver |
| 502 && testCase.output.unexpectedOutput && testCase.numRetries > 0) { | 502 && testCase.output.unexpectedOutput && testCase.numRetries > 0) { |
| 503 // Selenium tests can be flaky. Try rerunning. | 503 // Selenium tests can be flaky. Try rerunning. |
| 504 testCase.output.requestRetry = true; | 504 testCase.output.requestRetry = true; |
| 505 } | 505 } |
| 506 if (testCase.output.requestRetry) { | 506 if (testCase.output.requestRetry) { |
| 507 testCase.output.requestRetry = false; | 507 testCase.output.requestRetry = false; |
| 508 this.timedOut = false; | 508 this.timedOut = false; |
| 509 testCase.dynamic.numRetries--; | 509 testCase.dynamic.numRetries--; |
| 510 print("Potential flake. Re-running ${testCase.displayName} " + | 510 print("Potential flake. Re-running ${testCase.displayName} " + |
| 511 "(${testCase.dynamic.numRetries} attempt(s) remains)"); | 511 "(${testCase.dynamic.numRetries} attempt(s) remains)"); |
| (...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 833 bool _startingServer = false; | 833 bool _startingServer = false; |
| 834 | 834 |
| 835 /** True if we find that there is already a selenium jar running. */ | 835 /** True if we find that there is already a selenium jar running. */ |
| 836 bool _seleniumAlreadyRunning = false; | 836 bool _seleniumAlreadyRunning = false; |
| 837 | 837 |
| 838 ProcessQueue(int this._maxProcesses, | 838 ProcessQueue(int this._maxProcesses, |
| 839 String progress, | 839 String progress, |
| 840 Date startTime, | 840 Date startTime, |
| 841 bool printTiming, | 841 bool printTiming, |
| 842 Function this._enqueueMoreWork, | 842 Function this._enqueueMoreWork, |
| 843 [bool this._verbose = false, | 843 [bool verbose = false, |
| 844 bool this._listTests = false, | 844 bool listTests = false, |
| 845 bool this._keepGeneratedTests = false]) | 845 bool keepGeneratedTests = false]) |
| 846 : _tests = new Queue<TestCase>(), | 846 : _verbose = verbose, |
| 847 _listTests = listTests, |
| 848 _keepGeneratedTests = keepGeneratedTests, |
| 849 _tests = new Queue<TestCase>(), |
| 847 _progress = new ProgressIndicator.fromName(progress, | 850 _progress = new ProgressIndicator.fromName(progress, |
| 848 startTime, | 851 startTime, |
| 849 printTiming), | 852 printTiming), |
| 850 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), | 853 _batchProcesses = new Map<String, List<BatchRunnerProcess>>(), |
| 851 _testCache = new Map<String, List<TestInformation>>() { | 854 _testCache = new Map<String, List<TestInformation>>() { |
| 852 if (!_enqueueMoreWork(this)) _progress.allDone(); | 855 if (!_enqueueMoreWork(this)) _progress.allDone(); |
| 853 } | 856 } |
| 854 | 857 |
| 855 /** | 858 /** |
| 856 * Registers a TestSuite so that all of its tests will be run. | 859 * Registers a TestSuite so that all of its tests will be run. |
| (...skipping 251 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1108 // the developer doesn't waste his or her time trying to fix a bunch of | 1111 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1109 // tests that appear to be broken but were actually just flakes that | 1112 // tests that appear to be broken but were actually just flakes that |
| 1110 // didn't get retried because there had already been one failure. | 1113 // didn't get retried because there had already been one failure. |
| 1111 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1114 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1112 new RunningProcess(test, allowRetry, this).start(); | 1115 new RunningProcess(test, allowRetry, this).start(); |
| 1113 } | 1116 } |
| 1114 _numProcesses++; | 1117 _numProcesses++; |
| 1115 } | 1118 } |
| 1116 } | 1119 } |
| 1117 } | 1120 } |
| OLD | NEW |