| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 260 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 271 } | 271 } |
| 272 | 272 |
| 273 // Reverse result of a negative test. | 273 // Reverse result of a negative test. |
| 274 bool get hasFailed() { | 274 bool get hasFailed() { |
| 275 // TODO(efortuna): This is a total hack to keep our buildbots (more) green | 275 // TODO(efortuna): This is a total hack to keep our buildbots (more) green |
| 276 // while the VM team solves Issue 2124. Remove when issue is fixed. | 276 // while the VM team solves Issue 2124. Remove when issue is fixed. |
| 277 if (new Platform().operatingSystem() == 'windows' && exitCode == 253) { | 277 if (new Platform().operatingSystem() == 'windows' && exitCode == 253) { |
| 278 for (String line in testCase.output.stdout) { | 278 for (String line in testCase.output.stdout) { |
| 279 if (line.startsWith('VM exited with signal 1073741819')) { | 279 if (line.startsWith('VM exited with signal 1073741819')) { |
| 280 if (!alreadyPrintedWarning) { | 280 if (!alreadyPrintedWarning) { |
| 281 print("WARNING: VM crashed on this test with signal 1073741819. " + | 281 print("WARNING: VM crashed on this test with signal 1073741819. " + |
| 282 "This is a fake pass!!"); | 282 "This is a fake pass!!"); |
| 283 alreadyPrintedWarning = true; | 283 alreadyPrintedWarning = true; |
| 284 } | 284 } |
| 285 return testCase.expectedOutcomes.iterator().next() == FAIL; | 285 return testCase.expectedOutcomes.iterator().next() == FAIL; |
| 286 } | 286 } |
| 287 } | 287 } |
| 288 } | 288 } |
| 289 return (testCase.isNegative ? !didFail : didFail); | 289 return (testCase.isNegative ? !didFail : didFail); |
| 290 } | 290 } |
| 291 | 291 |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 376 return _didMultitestFail(errors, staticWarnings); | 376 return _didMultitestFail(errors, staticWarnings); |
| 377 } | 377 } |
| 378 return _didStandardTestFail(errors, staticWarnings); | 378 return _didStandardTestFail(errors, staticWarnings); |
| 379 } | 379 } |
| 380 | 380 |
| 381 bool _didMultitestFail(List errors, List staticWarnings) { | 381 bool _didMultitestFail(List errors, List staticWarnings) { |
| 382 Set<String> outcome = testCase.info.multitestOutcome; | 382 Set<String> outcome = testCase.info.multitestOutcome; |
| 383 Expect.isNotNull(outcome); | 383 Expect.isNotNull(outcome); |
| 384 if (outcome.contains('compile-time error') && errors.length > 0) { | 384 if (outcome.contains('compile-time error') && errors.length > 0) { |
| 385 return true; | 385 return true; |
| 386 } else if (outcome.contains('static type warning') | 386 } else if (outcome.contains('static type warning') |
| 387 && staticWarnings.length > 0) { | 387 && staticWarnings.length > 0) { |
| 388 return true; | 388 return true; |
| 389 } else if (outcome.isEmpty() | 389 } else if (outcome.isEmpty() |
| 390 && (errors.length > 0 || staticWarnings.length > 0)) { | 390 && (errors.length > 0 || staticWarnings.length > 0)) { |
| 391 return true; | 391 return true; |
| 392 } | 392 } |
| 393 return false; | 393 return false; |
| 394 } | 394 } |
| 395 | 395 |
| 396 bool _didStandardTestFail(List errors, List staticWarnings) { | 396 bool _didStandardTestFail(List errors, List staticWarnings) { |
| (...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 521 */ | 521 */ |
| 522 void testComplete(int exitCode) { | 522 void testComplete(int exitCode) { |
| 523 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, | 523 new TestOutput.fromCase(testCase, exitCode, timedOut, stdout, |
| 524 stderr, new Date.now().difference(startTime)); | 524 stderr, new Date.now().difference(startTime)); |
| 525 timeoutTimer.cancel(); | 525 timeoutTimer.cancel(); |
| 526 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { | 526 if (testCase.output.unexpectedOutput && testCase.configuration['verbose']) { |
| 527 print(testCase.displayName); | 527 print(testCase.displayName); |
| 528 for (var line in testCase.output.stderr) print(line); | 528 for (var line in testCase.output.stderr) print(line); |
| 529 for (var line in testCase.output.stdout) print(line); | 529 for (var line in testCase.output.stdout) print(line); |
| 530 } | 530 } |
| 531 if (allowRetries && testCase.usesWebDriver | 531 if (allowRetries && testCase.usesWebDriver |
| 532 && testCase.output.unexpectedOutput | 532 && testCase.output.unexpectedOutput |
| 533 && testCase.dynamic.numRetries > 0) { | 533 && testCase.dynamic.numRetries > 0) { |
| 534 // Selenium tests can be flaky. Try rerunning. | 534 // Selenium tests can be flaky. Try rerunning. |
| 535 testCase.output.requestRetry = true; | 535 testCase.output.requestRetry = true; |
| 536 } | 536 } |
| 537 if (testCase.output.requestRetry) { | 537 if (testCase.output.requestRetry) { |
| 538 testCase.output.requestRetry = false; | 538 testCase.output.requestRetry = false; |
| 539 this.timedOut = false; | 539 this.timedOut = false; |
| 540 testCase.dynamic.numRetries--; | 540 testCase.dynamic.numRetries--; |
| 541 print("Potential flake. Re-running ${testCase.displayName} " + | 541 print("Potential flake. Re-running ${testCase.displayName} " + |
| 542 "(${testCase.dynamic.numRetries} attempt(s) remains)"); | 542 "(${testCase.dynamic.numRetries} attempt(s) remains)"); |
| (...skipping 11 matching lines...) Expand all Loading... |
| 554 void stepExitHandler(int exitCode) { | 554 void stepExitHandler(int exitCode) { |
| 555 process.close(); | 555 process.close(); |
| 556 int totalSteps = testCase.commands.length; | 556 int totalSteps = testCase.commands.length; |
| 557 String suffix =' (step $currentStep of $totalSteps)'; | 557 String suffix =' (step $currentStep of $totalSteps)'; |
| 558 if (currentStep == totalSteps) { // done with test command | 558 if (currentStep == totalSteps) { // done with test command |
| 559 testComplete(exitCode); | 559 testComplete(exitCode); |
| 560 } else if (exitCode != 0) { | 560 } else if (exitCode != 0) { |
| 561 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); | 561 stderr.add('test.dart: Compilation failed$suffix, exit code $exitCode\n'); |
| 562 testComplete(exitCode); | 562 testComplete(exitCode); |
| 563 } else { | 563 } else { |
| 564 stderr.add('test.dart: Compilion finished $suffix\n'); | 564 stderr.add('test.dart: Compilation finished $suffix\n'); |
| 565 stdout.add('test.dart: Compilion finished $suffix\n'); | 565 stdout.add('test.dart: Compilation finished $suffix\n'); |
| 566 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && | 566 if (currentStep == totalSteps - 1 && testCase.usesWebDriver && |
| 567 !testCase.configuration['noBatch']) { | 567 !testCase.configuration['noBatch']) { |
| 568 // Note: processQueue will always be non-null for runtime == ie, ff, | 568 // Note: processQueue will always be non-null for runtime == ie, ff, |
| 569 // safari, chrome, opera. (It is only null for runtime == vm) | 569 // safari, chrome, opera. (It is only null for runtime == vm) |
| 570 processQueue._getBatchRunner(testCase).startTest(testCase); | 570 processQueue._getBatchRunner(testCase).startTest(testCase); |
| 571 } else { | 571 } else { |
| 572 runCommand(testCase.commands[currentStep++], stepExitHandler); | 572 runCommand(testCase.commands[currentStep++], stepExitHandler); |
| 573 } | 573 } |
| 574 } | 574 } |
| 575 } | 575 } |
| (...skipping 559 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1135 // the developer doesn't waste his or her time trying to fix a bunch of | 1135 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1136 // tests that appear to be broken but were actually just flakes that | 1136 // tests that appear to be broken but were actually just flakes that |
| 1137 // didn't get retried because there had already been one failure. | 1137 // didn't get retried because there had already been one failure. |
| 1138 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1138 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1139 new RunningProcess(test, allowRetry, this).start(); | 1139 new RunningProcess(test, allowRetry, this).start(); |
| 1140 } | 1140 } |
| 1141 _numProcesses++; | 1141 _numProcesses++; |
| 1142 } | 1142 } |
| 1143 } | 1143 } |
| 1144 } | 1144 } |
| OLD | NEW |