Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| 11 */ | 11 */ |
| 12 #library("test_runner"); | 12 #library("test_runner"); |
| 13 | 13 |
| 14 #import("dart:io"); | 14 #import("dart:io"); |
| 15 #import("dart:builtin"); | |
| 15 #import("status_file_parser.dart"); | 16 #import("status_file_parser.dart"); |
| 16 #import("test_progress.dart"); | 17 #import("test_progress.dart"); |
| 17 #import("test_suite.dart"); | 18 #import("test_suite.dart"); |
| 18 | 19 |
| 19 final int NO_TIMEOUT = 0; | 20 final int NO_TIMEOUT = 0; |
| 20 | 21 |
| 21 /** A command executed as a step in a test case. */ | 22 /** A command executed as a step in a test case. */ |
| 22 class Command { | 23 class Command { |
| 23 /** Path to the executable of this command. */ | 24 /** Path to the executable of this command. */ |
| 24 String executable; | 25 String executable; |
| (...skipping 27 matching lines...) Expand all Loading... | |
| 52 * The TestCase has a callback function, [completedHandler], that is run when | 53 * The TestCase has a callback function, [completedHandler], that is run when |
| 53 * the test is completed. | 54 * the test is completed. |
| 54 */ | 55 */ |
| 55 class TestCase { | 56 class TestCase { |
| 56 /** | 57 /** |
| 57 * A list of commands to execute. Most test cases have a single command. Frog | 58 * A list of commands to execute. Most test cases have a single command. Frog |
| 58 * tests have two commands, one to compilate the source and another to execute | 59 * tests have two commands, one to compilate the source and another to execute |
| 59 * it. Some isolate tests might even have three, if they require compiling | 60 * it. Some isolate tests might even have three, if they require compiling |
| 60 * multiple sources that are run in isolation. | 61 * multiple sources that are run in isolation. |
| 61 */ | 62 */ |
| 62 final List<Command> commands; | 63 List<Command> commands; |
| 63 | 64 |
| 64 Map configuration; | 65 Map configuration; |
| 65 String displayName; | 66 String displayName; |
| 66 TestOutput output; | 67 TestOutput output; |
| 67 bool isNegative; | 68 bool isNegative; |
| 68 Set<String> expectedOutcomes; | 69 Set<String> expectedOutcomes; |
| 69 Function completedHandler; | 70 Function completedHandler; |
| 70 TestInformation info; | 71 TestInformation info; |
| 71 | 72 |
| 72 TestCase(this.displayName, | 73 TestCase(this.displayName, |
| (...skipping 25 matching lines...) Expand all Loading... | |
| 98 | 99 |
| 99 if (prefix.length > 0) { | 100 if (prefix.length > 0) { |
| 100 var prefixSplit = prefix.split(' '); | 101 var prefixSplit = prefix.split(' '); |
| 101 newExecutablePath = prefixSplit[0]; | 102 newExecutablePath = prefixSplit[0]; |
| 102 for (int i = 1; i < prefixSplit.length; i++) { | 103 for (int i = 1; i < prefixSplit.length; i++) { |
| 103 var current = prefixSplit[i]; | 104 var current = prefixSplit[i]; |
| 104 if (!current.isEmpty()) newArguments.add(current); | 105 if (!current.isEmpty()) newArguments.add(current); |
| 105 } | 106 } |
| 106 newArguments.add(c.executable); | 107 newArguments.add(c.executable); |
| 107 } | 108 } |
| 108 newArguments.addAll(arguments); | 109 newArguments.addAll(c.arguments); |
| 109 var suffixSplit = suffix.split(' '); | 110 var suffixSplit = suffix.split(' '); |
| 110 suffixSplit.forEach((e) { | 111 suffixSplit.forEach((e) { |
| 111 if (!e.isEmpty()) newArguments.add(e); | 112 if (!e.isEmpty()) newArguments.add(e); |
| 112 }); | 113 }); |
| 113 final newCommand = new Command(newExecutablePath, newArguments); | 114 final newCommand = new Command(newExecutablePath, newArguments); |
| 114 newCommands.add(newCommand); | 115 newCommands.add(newCommand); |
| 115 Expect.stringEquals('$prefix ${c.commandLine} $suffix', | 116 Expect.stringEquals('$prefix ${c.commandLine} $suffix', |
| 116 newCommand.commandLine); | 117 newCommand.commandLine); |
| 117 } | 118 } |
| 118 commands = newCommand; | 119 commands = newCommands; |
| 119 } | 120 } |
| 120 } | 121 } |
| 121 | 122 |
| 122 int get timeout() => configuration['timeout']; | 123 int get timeout() => configuration['timeout']; |
| 123 | 124 |
| 124 String get configurationString() { | 125 String get configurationString() { |
| 125 final component = configuration['component']; | 126 final component = configuration['component']; |
| 126 final mode = configuration['mode']; | 127 final mode = configuration['mode']; |
| 127 final arch = configuration['arch']; | 128 final arch = configuration['arch']; |
| 128 return "$component ${mode}_$arch"; | 129 return "$component ${mode}_$arch"; |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 177 | 178 |
| 178 String get result(); | 179 String get result(); |
| 179 | 180 |
| 180 bool get unexpectedOutput(); | 181 bool get unexpectedOutput(); |
| 181 | 182 |
| 182 bool get hasCrashed(); | 183 bool get hasCrashed(); |
| 183 | 184 |
| 184 bool get hasTimedOut(); | 185 bool get hasTimedOut(); |
| 185 | 186 |
| 186 bool get didFail(); | 187 bool get didFail(); |
| 188 | |
| 189 Duration get time(); | |
| 190 | |
| 191 List<String> get stdout(); | |
| 187 | 192 |
| 193 List<String> get stderr(); | |
| 194 | |
| 188 List<String> get diagnostics(); | 195 List<String> get diagnostics(); |
| 189 } | 196 } |
| 190 | 197 |
| 191 class TestOutputImpl implements TestOutput { | 198 class TestOutputImpl implements TestOutput { |
| 192 TestCase testCase; | 199 TestCase testCase; |
| 193 int exitCode; | 200 int exitCode; |
| 194 bool timedOut; | 201 bool timedOut; |
| 195 bool failed = false; | 202 bool failed = false; |
| 196 List<String> stdout; | 203 List<String> stdout; |
| 197 List<String> stderr; | 204 List<String> stderr; |
| 198 Duration time; | 205 Duration time; |
| 199 List<String> diagnostics; | 206 List<String> diagnostics; |
| 200 | 207 |
| 201 /** | 208 /** |
| 202 * Set to true if we encounter a condition in the output that indicates we | 209 * Set to true if we encounter a condition in the output that indicates we |
| 203 * need to rerun this test. | 210 * need to rerun this test. |
| 204 */ | 211 */ |
| 205 bool requestRetry = false; | 212 bool requestRetry = false; |
| 206 | 213 |
| 207 // Don't call this constructor, call TestOutput.fromCase() to | 214 // Don't call this constructor, call TestOutput.fromCase() to |
| 208 // get anew TestOutput instance. | 215 // get anew TestOutput instance. |
| 209 TestOutputImpl(this.testCase, this.exitCode, this.timedOut, this.stdout, | 216 TestOutputImpl(TestCase this.testCase, |
| 210 this.stderr, this.time) { | 217 int this.exitCode, |
| 218 bool this.timedOut, | |
| 219 List<String> this.stdout, | |
| 220 List<String> this.stderr, | |
| 221 Duration this.time) { | |
| 211 testCase.output = this; | 222 testCase.output = this; |
| 212 diagnostics = []; | 223 diagnostics = []; |
| 213 } | 224 } |
| 214 | 225 |
| 215 factory TestOutputImpl.fromCase (testCase, exitCode, timedOut, | 226 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu t, |
| 216 stdout, stderr, time) { | 227 List<String> stdout, List<String> stderr, Dur ation time) { |
| 217 if (testCase is BrowserTestCase) { | 228 if (testCase is BrowserTestCase) { |
| 218 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, | 229 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, |
| 219 stdout, stderr, time); | 230 stdout, stderr, time); |
| 220 } else if (testCase.configuration['component'] == 'dartc') { | 231 } else if (testCase.configuration['component'] == 'dartc') { |
| 221 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, | 232 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, |
| 222 stdout, stderr, time); | 233 stdout, stderr, time); |
| 223 } | 234 } |
| 224 return new TestOutputImpl(testCase, exitCode, timedOut, | 235 return new TestOutputImpl(testCase, exitCode, timedOut, |
| 225 stdout, stderr, time); | 236 stdout, stderr, time); |
| 226 } | 237 } |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 258 class BrowserTestOutputImpl extends TestOutputImpl { | 269 class BrowserTestOutputImpl extends TestOutputImpl { |
| 259 BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : | 270 BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : |
| 260 super(testCase, exitCode, timedOut, stdout, stderr, time); | 271 super(testCase, exitCode, timedOut, stdout, stderr, time); |
| 261 | 272 |
| 262 bool get didFail() { | 273 bool get didFail() { |
| 263 // Browser case: | 274 // Browser case: |
| 264 // If the browser test failed, it may have been because DumpRenderTree | 275 // If the browser test failed, it may have been because DumpRenderTree |
| 265 // and the virtual framebuffer X server didn't hook up, or DRT crashed with | 276 // and the virtual framebuffer X server didn't hook up, or DRT crashed with |
| 266 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, | 277 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, |
| 267 // so we have to do this check first. | 278 // so we have to do this check first. |
| 268 for (String line in stderr) { | 279 for (String line in super.stderr) { |
|
zundel
2012/03/12 15:35:27
This is really tricky, overriding a field/getter i
zundel
2012/03/12 16:40:19
Nm, this isn't the problem I thought it was. I hi
Bill Hesse
2012/03/12 17:28:10
This class and the superclass are supposed to have
| |
| 269 if (line.contains('Gtk-WARNING **: cannot open display: :99') || | 280 if (line.contains('Gtk-WARNING **: cannot open display: :99') || |
| 270 line.contains('Failed to run command. return code=1')) { | 281 line.contains('Failed to run command. return code=1')) { |
| 271 // If we get the X server error, or DRT crashes with a core dump, retry | 282 // If we get the X server error, or DRT crashes with a core dump, retry |
| 272 // the test. | 283 // the test. |
| 273 if (testCase.dynamic.numRetries > 0) { | 284 if (testCase.dynamic.numRetries > 0) { |
| 274 requestRetry = true; | 285 requestRetry = true; |
| 275 } | 286 } |
| 276 return true; | 287 return true; |
| 277 } | 288 } |
| 278 } | 289 } |
| 279 | 290 |
| 280 // Browser tests fail unless stdout contains | 291 // Browser tests fail unless stdout contains |
| 281 // 'Content-Type: text/plain\nPASS'. | 292 // 'Content-Type: text/plain\nPASS'. |
| 282 String previous_line = ''; | 293 String previous_line = ''; |
| 283 for (String line in stdout) { | 294 for (String line in super.stdout) { |
| 284 if (line == 'PASS' && previous_line == 'Content-Type: text/plain') { | 295 if (line == 'PASS' && previous_line == 'Content-Type: text/plain') { |
| 285 return (exitCode != 0 && !hasCrashed); | 296 return (exitCode != 0 && !hasCrashed); |
| 286 } | 297 } |
| 287 previous_line = line; | 298 previous_line = line; |
| 288 } | 299 } |
| 289 return true; | 300 return true; |
| 290 } | 301 } |
| 291 } | 302 } |
| 292 | 303 |
| 293 // The static analyzer does not actaully execute code, so | 304 // The static analyzer does not actually execute code, so |
| 294 // the criteria for success now depend on the text sent | 305 // the criteria for success now depend on the text sent |
| 295 // to stderr. | 306 // to stderr. |
| 296 class AnalysisTestOutputImpl extends TestOutputImpl { | 307 class AnalysisTestOutputImpl extends TestOutputImpl { |
| 297 bool alreadyComputed = false; | 308 bool alreadyComputed = false; |
| 298 bool failResult; | 309 bool failResult; |
| 299 AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : | 310 AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : |
| 300 super(testCase, exitCode, timedOut, stdout, stderr, time) { | 311 super(testCase, exitCode, timedOut, stdout, stderr, time) { |
| 301 } | 312 } |
| 302 | 313 |
| 303 bool get didFail() { | 314 bool get didFail() { |
| 304 if (!alreadyComputed) { | 315 if (!alreadyComputed) { |
| 305 failResult = _didFail(); | 316 failResult = _didFail(); |
| 306 alreadyComputed = true; | 317 alreadyComputed = true; |
| 307 } | 318 } |
| 308 return failResult; | 319 return failResult; |
| 309 } | 320 } |
| 310 | 321 |
| 311 bool _didFail() { | 322 bool _didFail() { |
| 312 if (hasCrashed) return false; | 323 if (hasCrashed) return false; |
| 313 | 324 |
| 314 List<String> errors = []; | 325 List<String> errors = []; |
| 315 List<String> staticWarnings = []; | 326 List<String> staticWarnings = []; |
| 316 | 327 |
| 317 // Read the returned list of errors and stuff them away. | 328 // Read the returned list of errors and stuff them away. |
| 318 for (String line in stderr) { | 329 for (String line in super.stderr) { |
| 319 if (line.length == 0) continue; | 330 if (line.length == 0) continue; |
| 320 List<String> fields = splitMachineError(line); | 331 List<String> fields = splitMachineError(line); |
| 321 if (fields[0] == 'ERROR') { | 332 if (fields[0] == 'ERROR') { |
| 322 errors.add(fields); | 333 errors.add(fields); |
| 323 } else if (fields[0] == 'WARNING') { | 334 } else if (fields[0] == 'WARNING') { |
| 324 // We only care about testing Static type warnings | 335 // We only care about testing Static type warnings |
| 325 // ignore all others | 336 // ignore all others |
| 326 if (fields[1] == 'STATIC_TYPE') { | 337 if (fields[1] == 'STATIC_TYPE') { |
| 327 staticWarnings.add(fields); | 338 staticWarnings.add(fields); |
| 328 } | 339 } |
| (...skipping 667 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 996 * Java server. | 1007 * Java server. |
| 997 */ | 1008 */ |
| 998 void _startSeleniumServer() { | 1009 void _startSeleniumServer() { |
| 999 // Get the absolute path to the Selenium jar. | 1010 // Get the absolute path to the Selenium jar. |
| 1000 String filePath = new Options().script; | 1011 String filePath = new Options().script; |
| 1001 String pathSep = new Platform().pathSeparator(); | 1012 String pathSep = new Platform().pathSeparator(); |
| 1002 int index = filePath.lastIndexOf(pathSep); | 1013 int index = filePath.lastIndexOf(pathSep); |
| 1003 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; | 1014 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; |
| 1004 var dir = new Directory(filePath); | 1015 var dir = new Directory(filePath); |
| 1005 dir.onFile = (String file) { | 1016 dir.onFile = (String file) { |
| 1006 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) | 1017 if (const RegExp("selenium-server-standalone-.*\.jar").hasMatch(file) |
|
Bill Hesse
2012/03/12 17:28:10
This is a stray typo.
This accidentally made it i
| |
| 1007 && _seleniumServer == null) { | 1018 && _seleniumServer == null) { |
| 1008 _seleniumServer = new Process.start('java', ['-jar', file]); | 1019 _seleniumServer = new Process.start('java', ['-jar', file]); |
| 1009 // Heads up: there seems to an obscure data race of some form in | 1020 // Heads up: there seems to an obscure data race of some form in |
| 1010 // the VM between launching the server process and launching the test | 1021 // the VM between launching the server process and launching the test |
| 1011 // tasks that disappears when you read IO (which is convenient, since | 1022 // tasks that disappears when you read IO (which is convenient, since |
| 1012 // that is our condition for knowing that the server is ready). | 1023 // that is our condition for knowing that the server is ready). |
| 1013 StringInputStream stdoutStringStream = | 1024 StringInputStream stdoutStringStream = |
| 1014 new StringInputStream(_seleniumServer.stdout); | 1025 new StringInputStream(_seleniumServer.stdout); |
| 1015 StringInputStream stderrStringStream = | 1026 StringInputStream stderrStringStream = |
| 1016 new StringInputStream(_seleniumServer.stderr); | 1027 new StringInputStream(_seleniumServer.stderr); |
| (...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1094 // the developer doesn't waste his or her time trying to fix a bunch of | 1105 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1095 // tests that appear to be broken but were actually just flakes that | 1106 // tests that appear to be broken but were actually just flakes that |
| 1096 // didn't get retried because there had already been one failure. | 1107 // didn't get retried because there had already been one failure. |
| 1097 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1108 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1098 new RunningProcess(test, allowRetry, this).start(); | 1109 new RunningProcess(test, allowRetry, this).start(); |
| 1099 } | 1110 } |
| 1100 _numProcesses++; | 1111 _numProcesses++; |
| 1101 } | 1112 } |
| 1102 } | 1113 } |
| 1103 } | 1114 } |
| OLD | NEW |