Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #library("test_runner"); | 5 #library("test_runner"); |
| 6 | 6 |
| 7 #import("dart:io"); | 7 #import("dart:io"); |
| 8 #import("status_file_parser.dart"); | 8 #import("status_file_parser.dart"); |
| 9 #import("test_progress.dart"); | 9 #import("test_progress.dart"); |
| 10 #import("test_suite.dart"); | 10 #import("test_suite.dart"); |
| 11 | 11 |
| 12 /** | 12 /** |
| 13 * Classes and methods for executing tests. | 13 * Classes and methods for executing tests. |
| 14 * | 14 * |
| 15 * This module includes: | 15 * This module includes: |
| 16 * - Managing parallel execution of tests, including timeout checks. | 16 * - Managing parallel execution of tests, including timeout checks. |
| 17 * - Evaluating the output of each test as pass/fail/crash/timeout. | 17 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| 18 */ | 18 */ |
| 19 | 19 |
| 20 final int NO_TIMEOUT = 0; | 20 final int NO_TIMEOUT = 0; |
| 21 | 21 |
| 22 | 22 |
| 23 /** | |
| 24 * [TestCase] contains all the information needed to run a test and evaluate | |
| 25 * its output. Running a test involves starting a separate process, with | |
|
Bob Nystrom
2012/02/07 18:41:24
Style nit, but editors (people, not software) thes
Bill Hesse
2012/02/09 15:47:39
But they assume everything is in a proportional fo
| |
| 26 * the executable and arguments given by the TestCase, and recording its | |
| 27 * stdout and stderr output streams, and its exit code. TestCase only | |
| 28 * contains static information about the test; actually running the test is | |
| 29 * performed by [ProcessQueue] using a [RunningProcess] object. | |
| 30 * | |
| 31 * The output information is stored in a [TestOutput] instance contained | |
| 32 * in the TestCase. The [TestOutput] instance is responsible for evaluating | |
|
Bob Nystrom
2012/02/07 18:41:24
I would probably only put the first occurrence of
| |
| 33 * if the test has passed, failed, crashed, or timed out, and the TestCase | |
| 34 * has information about what the expected result of the test should be. | |
| 35 * | |
| 36 * The TestCase has a callback function, completedHandler, that is run when | |
|
Bob Nystrom
2012/02/07 18:41:24
[completedHandler]
| |
| 37 * the test is completed. | |
| 38 */ | |
| 23 class TestCase { | 39 class TestCase { |
| 24 String executablePath; | 40 String executablePath; |
| 25 List<String> arguments; | 41 List<String> arguments; |
| 26 Map configuration; | 42 Map configuration; |
| 27 String commandLine; | 43 String commandLine; |
| 28 String displayName; | 44 String displayName; |
| 29 TestOutput output; | 45 TestOutput output; |
| 30 bool isNegative; | 46 bool isNegative; |
| 31 Set<String> expectedOutcomes; | 47 Set<String> expectedOutcomes; |
| 32 Function completedHandler; | 48 Function completedHandler; |
| (...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 82 final mode = configuration['mode']; | 98 final mode = configuration['mode']; |
| 83 final arch = configuration['arch']; | 99 final arch = configuration['arch']; |
| 84 return "$component ${mode}_$arch"; | 100 return "$component ${mode}_$arch"; |
| 85 } | 101 } |
| 86 | 102 |
| 87 void completed() { completedHandler(this); } | 103 void completed() { completedHandler(this); } |
| 88 } | 104 } |
| 89 | 105 |
| 90 | 106 |
| 91 /** | 107 /** |
| 92 * BrowserTestCase has an extra compilation command that is run by | 108 * BrowserTestCase has an extra compilation command that is run in a separate |
| 93 * RunningProcess.start(), and it checks conditions on the test output | 109 * process, before the regular test is run as in the base class [TestCase]. |
| 94 * in TestOutput.didFail(). | 110 * If the compilation command fails, then the rest of the test is not run. |
| 95 */ | 111 */ |
| 96 class BrowserTestCase extends TestCase { | 112 class BrowserTestCase extends TestCase { |
| 113 /** | |
| 114 * The executable that is run in a new process in the compilation phase. | |
| 115 */ | |
| 97 String compilerPath; | 116 String compilerPath; |
| 117 /** | |
| 118 * The arguments for the compilation command. | |
| 119 */ | |
| 98 List<String> compilerArguments; | 120 List<String> compilerArguments; |
| 99 | 121 |
| 100 BrowserTestCase(displayName, | 122 BrowserTestCase(displayName, |
| 101 this.compilerPath, | 123 this.compilerPath, |
| 102 this.compilerArguments, | 124 this.compilerArguments, |
| 103 executablePath, | 125 executablePath, |
| 104 arguments, | 126 arguments, |
| 105 configuration, | 127 configuration, |
| 106 completedHandler, | 128 completedHandler, |
| 107 expectedOutcomes, | 129 expectedOutcomes, |
| 108 [isNegative = false]) : super(displayName, | 130 [isNegative = false]) : super(displayName, |
| 109 executablePath, | 131 executablePath, |
| 110 arguments, | 132 arguments, |
| 111 configuration, | 133 configuration, |
| 112 completedHandler, | 134 completedHandler, |
| 113 expectedOutcomes, | 135 expectedOutcomes, |
| 114 isNegative) { | 136 isNegative) { |
| 115 if (compilerPath != null) { | 137 if (compilerPath != null) { |
| 116 commandLine = 'execution command: $commandLine'; | 138 commandLine = 'execution command: $commandLine'; |
| 117 String compilationCommand = | 139 String compilationCommand = |
| 118 '$compilerPath ${Strings.join(compilerArguments, " ")}'; | 140 '$compilerPath ${Strings.join(compilerArguments, " ")}'; |
| 119 commandLine = 'compilation command: $compilationCommand\n$commandLine'; | 141 commandLine = 'compilation command: $compilationCommand\n$commandLine'; |
| 120 } | 142 } |
| 121 } | 143 } |
| 122 } | 144 } |
| 123 | 145 |
| 124 | 146 |
| 147 /** | |
| 148 * TestOutput records the output of a completed test: the process's exit code, | |
| 149 * the standard output and standard error, whether the process timed out, and | |
| 150 * the time the process took to run. It also contains a pointer to the | |
| 151 * [TestCase] this is the output of. | |
| 152 */ | |
| 125 class TestOutput { | 153 class TestOutput { |
| 126 // The TestCase this is the output from. | |
| 127 TestCase testCase; | 154 TestCase testCase; |
| 128 int exitCode; | 155 int exitCode; |
| 129 bool timedOut; | 156 bool timedOut; |
| 130 bool failed = false; | 157 bool failed = false; |
| 131 List<String> stdout; | 158 List<String> stdout; |
| 132 List<String> stderr; | 159 List<String> stderr; |
| 133 Duration time; | 160 Duration time; |
| 134 | 161 |
| 135 TestOutput(this.testCase, this.exitCode, this.timedOut, this.stdout, | 162 TestOutput(this.testCase, this.exitCode, this.timedOut, this.stdout, |
| 136 this.stderr, this.time) { | 163 this.stderr, this.time) { |
| (...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 181 return testCase.isNegative; | 208 return testCase.isNegative; |
| 182 } | 209 } |
| 183 } | 210 } |
| 184 return true; | 211 return true; |
| 185 } | 212 } |
| 186 | 213 |
| 187 // Reverse result of a negative test. | 214 // Reverse result of a negative test. |
| 188 bool get hasFailed() => (testCase.isNegative ? !didFail : didFail); | 215 bool get hasFailed() => (testCase.isNegative ? !didFail : didFail); |
| 189 } | 216 } |
| 190 | 217 |
| 191 | 218 /** |
| 219 * A RunningProcess actually runs a test, getting the command lines from | |
| 220 * its [TestCase], starting the test process (and first, a compilation | |
| 221 * process if the [TestCase] is a [BrowserTestCase]), creating a timeout | |
| 222 * timer, and recording the results in a new [TestOutput] object, which it | |
| 223 * attaches to the [TestCase]. The lifetime of the RunningProcess is limited | |
| 224 * to the time it takes to start the process, run the process, and record | |
| 225 * the result; there are no pointers to it, so it should be available to | |
| 226 * be garbage collected as soon as it is done. | |
| 227 */ | |
| 192 class RunningProcess { | 228 class RunningProcess { |
| 193 Process process; | 229 Process process; |
| 194 TestCase testCase; | 230 TestCase testCase; |
| 195 bool timedOut = false; | 231 bool timedOut = false; |
| 196 Date startTime; | 232 Date startTime; |
| 197 Timer timeoutTimer; | 233 Timer timeoutTimer; |
| 198 List<String> stdout; | 234 List<String> stdout; |
| 199 List<String> stderr; | 235 List<String> stderr; |
| 200 List<Function> handlers; | 236 List<Function> handlers; |
| 201 | 237 |
| (...skipping 205 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 407 _testStdout = new List<String>(); | 443 _testStdout = new List<String>(); |
| 408 _testStderr = new List<String>(); | 444 _testStderr = new List<String>(); |
| 409 _stdoutStream.lineHandler = _readOutput(_stdoutStream, _testStdout); | 445 _stdoutStream.lineHandler = _readOutput(_stdoutStream, _testStdout); |
| 410 _stderrStream.lineHandler = _readOutput(_stderrStream, _testStderr); | 446 _stderrStream.lineHandler = _readOutput(_stderrStream, _testStderr); |
| 411 _process.exitHandler = _exitHandler; | 447 _process.exitHandler = _exitHandler; |
| 412 _process.startHandler = then; | 448 _process.startHandler = then; |
| 413 } | 449 } |
| 414 } | 450 } |
| 415 | 451 |
| 416 | 452 |
| 453 /** | |
| 454 * ProcessQueue is the master control class, responsible for running all | |
| 455 * the tests in all the [TestSuite]s that have been registered. It includes | |
| 456 * a rate-limited queue to run a limited number of tests in parallel, | |
| 457 * a ProgressIndicator which prints output when tests are started and | |
| 458 * and completed, and a summary report when all tests are completed, | |
| 459 * and counters to determine when all of the tests in all of the test suites | |
| 460 * have completed. | |
| 461 * | |
| 462 * [TestSuite] objects are registered with the ProcessQueue when it is | |
| 463 * created, and then the ProcessQueue requests them to enqueue their tests | |
| 464 * (asynchronously) by calling their "forEachTest" methods. | |
| 465 * | |
| 466 * Because multiple configurations may be run on each test suite, the | |
| 467 * ProcessQueue contains a cache in which a test suite may record information | |
| 468 * about its list of tests, and retrieve that information when it is called | |
|
Bob Nystrom
2012/02/07 18:41:24
"retrieves"
Bill Hesse
2012/02/09 15:47:39
this is short for "may retrieve". Adding "may"
| |
| 469 * upon to enqueue its tests again. | |
| 470 */ | |
| 417 class ProcessQueue { | 471 class ProcessQueue { |
| 418 int _numProcesses = 0; | 472 int _numProcesses = 0; |
| 419 int _activeTestListers = 0; | 473 int _activeTestListers = 0; |
| 420 int _maxProcesses; | 474 int _maxProcesses; |
| 421 bool _verbose; | 475 bool _verbose; |
| 422 bool _listTests; | 476 bool _listTests; |
| 423 bool _keepGeneratedTests; | 477 bool _keepGeneratedTests; |
| 424 Function _enqueueMoreWork; | 478 Function _enqueueMoreWork; |
| 425 Queue<TestCase> _tests; | 479 Queue<TestCase> _tests; |
| 426 ProgressIndicator _progress; | 480 ProgressIndicator _progress; |
| (...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 560 test.displayName != 'dartc/junit_tests') { | 614 test.displayName != 'dartc/junit_tests') { |
| 561 _ensureDartcBatchRunnersStarted(test.executablePath); | 615 _ensureDartcBatchRunnersStarted(test.executablePath); |
| 562 _getDartcBatchRunnerProcess().startTest(test); | 616 _getDartcBatchRunnerProcess().startTest(test); |
| 563 } else { | 617 } else { |
| 564 new RunningProcess(test).start(); | 618 new RunningProcess(test).start(); |
| 565 } | 619 } |
| 566 _numProcesses++; | 620 _numProcesses++; |
| 567 } | 621 } |
| 568 } | 622 } |
| 569 } | 623 } |
| OLD | NEW |