Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(411)

Side by Side Diff: tools/testing/dart/test_runner.dart

Issue 9662063: Fix some type and logical errors in test.dart, found by the Dart editor. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tools/testing/dart/test_progress.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /** 5 /**
6 * Classes and methods for executing tests. 6 * Classes and methods for executing tests.
7 * 7 *
8 * This module includes: 8 * This module includes:
9 * - Managing parallel execution of tests, including timeout checks. 9 * - Managing parallel execution of tests, including timeout checks.
10 * - Evaluating the output of each test as pass/fail/crash/timeout. 10 * - Evaluating the output of each test as pass/fail/crash/timeout.
11 */ 11 */
12 #library("test_runner"); 12 #library("test_runner");
13 13
14 #import("dart:io"); 14 #import("dart:io");
15 #import("dart:builtin");
15 #import("status_file_parser.dart"); 16 #import("status_file_parser.dart");
16 #import("test_progress.dart"); 17 #import("test_progress.dart");
17 #import("test_suite.dart"); 18 #import("test_suite.dart");
18 19
19 final int NO_TIMEOUT = 0; 20 final int NO_TIMEOUT = 0;
20 21
21 /** A command executed as a step in a test case. */ 22 /** A command executed as a step in a test case. */
22 class Command { 23 class Command {
23 /** Path to the executable of this command. */ 24 /** Path to the executable of this command. */
24 String executable; 25 String executable;
(...skipping 27 matching lines...) Expand all
52 * The TestCase has a callback function, [completedHandler], that is run when 53 * The TestCase has a callback function, [completedHandler], that is run when
53 * the test is completed. 54 * the test is completed.
54 */ 55 */
55 class TestCase { 56 class TestCase {
56 /** 57 /**
57 * A list of commands to execute. Most test cases have a single command. Frog 58 * A list of commands to execute. Most test cases have a single command. Frog
58 * tests have two commands, one to compilate the source and another to execute 59 * tests have two commands, one to compilate the source and another to execute
59 * it. Some isolate tests might even have three, if they require compiling 60 * it. Some isolate tests might even have three, if they require compiling
60 * multiple sources that are run in isolation. 61 * multiple sources that are run in isolation.
61 */ 62 */
62 final List<Command> commands; 63 List<Command> commands;
63 64
64 Map configuration; 65 Map configuration;
65 String displayName; 66 String displayName;
66 TestOutput output; 67 TestOutput output;
67 bool isNegative; 68 bool isNegative;
68 Set<String> expectedOutcomes; 69 Set<String> expectedOutcomes;
69 Function completedHandler; 70 Function completedHandler;
70 TestInformation info; 71 TestInformation info;
71 72
72 TestCase(this.displayName, 73 TestCase(this.displayName,
(...skipping 25 matching lines...) Expand all
98 99
99 if (prefix.length > 0) { 100 if (prefix.length > 0) {
100 var prefixSplit = prefix.split(' '); 101 var prefixSplit = prefix.split(' ');
101 newExecutablePath = prefixSplit[0]; 102 newExecutablePath = prefixSplit[0];
102 for (int i = 1; i < prefixSplit.length; i++) { 103 for (int i = 1; i < prefixSplit.length; i++) {
103 var current = prefixSplit[i]; 104 var current = prefixSplit[i];
104 if (!current.isEmpty()) newArguments.add(current); 105 if (!current.isEmpty()) newArguments.add(current);
105 } 106 }
106 newArguments.add(c.executable); 107 newArguments.add(c.executable);
107 } 108 }
108 newArguments.addAll(arguments); 109 newArguments.addAll(c.arguments);
109 var suffixSplit = suffix.split(' '); 110 var suffixSplit = suffix.split(' ');
110 suffixSplit.forEach((e) { 111 suffixSplit.forEach((e) {
111 if (!e.isEmpty()) newArguments.add(e); 112 if (!e.isEmpty()) newArguments.add(e);
112 }); 113 });
113 final newCommand = new Command(newExecutablePath, newArguments); 114 final newCommand = new Command(newExecutablePath, newArguments);
114 newCommands.add(newCommand); 115 newCommands.add(newCommand);
115 Expect.stringEquals('$prefix ${c.commandLine} $suffix', 116 Expect.stringEquals('$prefix ${c.commandLine} $suffix',
116 newCommand.commandLine); 117 newCommand.commandLine);
117 } 118 }
118 commands = newCommand; 119 commands = newCommands;
119 } 120 }
120 } 121 }
121 122
122 int get timeout() => configuration['timeout']; 123 int get timeout() => configuration['timeout'];
123 124
124 String get configurationString() { 125 String get configurationString() {
125 final component = configuration['component']; 126 final component = configuration['component'];
126 final mode = configuration['mode']; 127 final mode = configuration['mode'];
127 final arch = configuration['arch']; 128 final arch = configuration['arch'];
128 return "$component ${mode}_$arch"; 129 return "$component ${mode}_$arch";
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
177 178
178 String get result(); 179 String get result();
179 180
180 bool get unexpectedOutput(); 181 bool get unexpectedOutput();
181 182
182 bool get hasCrashed(); 183 bool get hasCrashed();
183 184
184 bool get hasTimedOut(); 185 bool get hasTimedOut();
185 186
186 bool get didFail(); 187 bool get didFail();
188
189 Duration get time();
190
191 List<String> get stdout();
187 192
193 List<String> get stderr();
194
188 List<String> get diagnostics(); 195 List<String> get diagnostics();
189 } 196 }
190 197
191 class TestOutputImpl implements TestOutput { 198 class TestOutputImpl implements TestOutput {
192 TestCase testCase; 199 TestCase testCase;
193 int exitCode; 200 int exitCode;
194 bool timedOut; 201 bool timedOut;
195 bool failed = false; 202 bool failed = false;
196 List<String> stdout; 203 List<String> stdout;
197 List<String> stderr; 204 List<String> stderr;
198 Duration time; 205 Duration time;
199 List<String> diagnostics; 206 List<String> diagnostics;
200 207
201 /** 208 /**
202 * Set to true if we encounter a condition in the output that indicates we 209 * Set to true if we encounter a condition in the output that indicates we
203 * need to rerun this test. 210 * need to rerun this test.
204 */ 211 */
205 bool requestRetry = false; 212 bool requestRetry = false;
206 213
207 // Don't call this constructor, call TestOutput.fromCase() to 214 // Don't call this constructor, call TestOutput.fromCase() to
208 // get anew TestOutput instance. 215 // get anew TestOutput instance.
209 TestOutputImpl(this.testCase, this.exitCode, this.timedOut, this.stdout, 216 TestOutputImpl(TestCase this.testCase,
210 this.stderr, this.time) { 217 int this.exitCode,
218 bool this.timedOut,
219 List<String> this.stdout,
220 List<String> this.stderr,
221 Duration this.time) {
211 testCase.output = this; 222 testCase.output = this;
212 diagnostics = []; 223 diagnostics = [];
213 } 224 }
214 225
215 factory TestOutputImpl.fromCase (testCase, exitCode, timedOut, 226 factory TestOutputImpl.fromCase (TestCase testCase, int exitCode, bool timedOu t,
216 stdout, stderr, time) { 227 List<String> stdout, List<String> stderr, Dur ation time) {
217 if (testCase is BrowserTestCase) { 228 if (testCase is BrowserTestCase) {
218 return new BrowserTestOutputImpl(testCase, exitCode, timedOut, 229 return new BrowserTestOutputImpl(testCase, exitCode, timedOut,
219 stdout, stderr, time); 230 stdout, stderr, time);
220 } else if (testCase.configuration['component'] == 'dartc') { 231 } else if (testCase.configuration['component'] == 'dartc') {
221 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut, 232 return new AnalysisTestOutputImpl(testCase, exitCode, timedOut,
222 stdout, stderr, time); 233 stdout, stderr, time);
223 } 234 }
224 return new TestOutputImpl(testCase, exitCode, timedOut, 235 return new TestOutputImpl(testCase, exitCode, timedOut,
225 stdout, stderr, time); 236 stdout, stderr, time);
226 } 237 }
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
258 class BrowserTestOutputImpl extends TestOutputImpl { 269 class BrowserTestOutputImpl extends TestOutputImpl {
259 BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : 270 BrowserTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) :
260 super(testCase, exitCode, timedOut, stdout, stderr, time); 271 super(testCase, exitCode, timedOut, stdout, stderr, time);
261 272
262 bool get didFail() { 273 bool get didFail() {
263 // Browser case: 274 // Browser case:
264 // If the browser test failed, it may have been because DumpRenderTree 275 // If the browser test failed, it may have been because DumpRenderTree
265 // and the virtual framebuffer X server didn't hook up, or DRT crashed with 276 // and the virtual framebuffer X server didn't hook up, or DRT crashed with
266 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, 277 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS,
267 // so we have to do this check first. 278 // so we have to do this check first.
268 for (String line in stderr) { 279 for (String line in super.stderr) {
zundel 2012/03/12 15:35:27 This is really tricky, overriding a field/getter i
zundel 2012/03/12 16:40:19 Nm, this isn't the problem I thought it was. I hi
Bill Hesse 2012/03/12 17:28:10 This class and the superclass are supposed to have
269 if (line.contains('Gtk-WARNING **: cannot open display: :99') || 280 if (line.contains('Gtk-WARNING **: cannot open display: :99') ||
270 line.contains('Failed to run command. return code=1')) { 281 line.contains('Failed to run command. return code=1')) {
271 // If we get the X server error, or DRT crashes with a core dump, retry 282 // If we get the X server error, or DRT crashes with a core dump, retry
272 // the test. 283 // the test.
273 if (testCase.dynamic.numRetries > 0) { 284 if (testCase.dynamic.numRetries > 0) {
274 requestRetry = true; 285 requestRetry = true;
275 } 286 }
276 return true; 287 return true;
277 } 288 }
278 } 289 }
279 290
280 // Browser tests fail unless stdout contains 291 // Browser tests fail unless stdout contains
281 // 'Content-Type: text/plain\nPASS'. 292 // 'Content-Type: text/plain\nPASS'.
282 String previous_line = ''; 293 String previous_line = '';
283 for (String line in stdout) { 294 for (String line in super.stdout) {
284 if (line == 'PASS' && previous_line == 'Content-Type: text/plain') { 295 if (line == 'PASS' && previous_line == 'Content-Type: text/plain') {
285 return (exitCode != 0 && !hasCrashed); 296 return (exitCode != 0 && !hasCrashed);
286 } 297 }
287 previous_line = line; 298 previous_line = line;
288 } 299 }
289 return true; 300 return true;
290 } 301 }
291 } 302 }
292 303
293 // The static analyzer does not actaully execute code, so 304 // The static analyzer does not actually execute code, so
294 // the criteria for success now depend on the text sent 305 // the criteria for success now depend on the text sent
295 // to stderr. 306 // to stderr.
296 class AnalysisTestOutputImpl extends TestOutputImpl { 307 class AnalysisTestOutputImpl extends TestOutputImpl {
297 bool alreadyComputed = false; 308 bool alreadyComputed = false;
298 bool failResult; 309 bool failResult;
299 AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) : 310 AnalysisTestOutputImpl(testCase, exitCode, timedOut, stdout, stderr, time) :
300 super(testCase, exitCode, timedOut, stdout, stderr, time) { 311 super(testCase, exitCode, timedOut, stdout, stderr, time) {
301 } 312 }
302 313
303 bool get didFail() { 314 bool get didFail() {
304 if (!alreadyComputed) { 315 if (!alreadyComputed) {
305 failResult = _didFail(); 316 failResult = _didFail();
306 alreadyComputed = true; 317 alreadyComputed = true;
307 } 318 }
308 return failResult; 319 return failResult;
309 } 320 }
310 321
311 bool _didFail() { 322 bool _didFail() {
312 if (hasCrashed) return false; 323 if (hasCrashed) return false;
313 324
314 List<String> errors = []; 325 List<String> errors = [];
315 List<String> staticWarnings = []; 326 List<String> staticWarnings = [];
316 327
317 // Read the returned list of errors and stuff them away. 328 // Read the returned list of errors and stuff them away.
318 for (String line in stderr) { 329 for (String line in super.stderr) {
319 if (line.length == 0) continue; 330 if (line.length == 0) continue;
320 List<String> fields = splitMachineError(line); 331 List<String> fields = splitMachineError(line);
321 if (fields[0] == 'ERROR') { 332 if (fields[0] == 'ERROR') {
322 errors.add(fields); 333 errors.add(fields);
323 } else if (fields[0] == 'WARNING') { 334 } else if (fields[0] == 'WARNING') {
324 // We only care about testing Static type warnings 335 // We only care about testing Static type warnings
325 // ignore all others 336 // ignore all others
326 if (fields[1] == 'STATIC_TYPE') { 337 if (fields[1] == 'STATIC_TYPE') {
327 staticWarnings.add(fields); 338 staticWarnings.add(fields);
328 } 339 }
(...skipping 667 matching lines...) Expand 10 before | Expand all | Expand 10 after
996 * Java server. 1007 * Java server.
997 */ 1008 */
998 void _startSeleniumServer() { 1009 void _startSeleniumServer() {
999 // Get the absolute path to the Selenium jar. 1010 // Get the absolute path to the Selenium jar.
1000 String filePath = new Options().script; 1011 String filePath = new Options().script;
1001 String pathSep = new Platform().pathSeparator(); 1012 String pathSep = new Platform().pathSeparator();
1002 int index = filePath.lastIndexOf(pathSep); 1013 int index = filePath.lastIndexOf(pathSep);
1003 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; 1014 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}';
1004 var dir = new Directory(filePath); 1015 var dir = new Directory(filePath);
1005 dir.onFile = (String file) { 1016 dir.onFile = (String file) {
1006 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) 1017 if (const RegExp("selenium-server-standalone-.*\.jar").hasMatch(file)
Bill Hesse 2012/03/12 17:28:10 This is a stray typo. This accidentally made it i
1007 && _seleniumServer == null) { 1018 && _seleniumServer == null) {
1008 _seleniumServer = new Process.start('java', ['-jar', file]); 1019 _seleniumServer = new Process.start('java', ['-jar', file]);
1009 // Heads up: there seems to an obscure data race of some form in 1020 // Heads up: there seems to an obscure data race of some form in
1010 // the VM between launching the server process and launching the test 1021 // the VM between launching the server process and launching the test
1011 // tasks that disappears when you read IO (which is convenient, since 1022 // tasks that disappears when you read IO (which is convenient, since
1012 // that is our condition for knowing that the server is ready). 1023 // that is our condition for knowing that the server is ready).
1013 StringInputStream stdoutStringStream = 1024 StringInputStream stdoutStringStream =
1014 new StringInputStream(_seleniumServer.stdout); 1025 new StringInputStream(_seleniumServer.stdout);
1015 StringInputStream stderrStringStream = 1026 StringInputStream stderrStringStream =
1016 new StringInputStream(_seleniumServer.stderr); 1027 new StringInputStream(_seleniumServer.stderr);
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
1094 // the developer doesn't waste his or her time trying to fix a bunch of 1105 // the developer doesn't waste his or her time trying to fix a bunch of
1095 // tests that appear to be broken but were actually just flakes that 1106 // tests that appear to be broken but were actually just flakes that
1096 // didn't get retried because there had already been one failure. 1107 // didn't get retried because there had already been one failure.
1097 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; 1108 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests;
1098 new RunningProcess(test, allowRetry, this).start(); 1109 new RunningProcess(test, allowRetry, this).start();
1099 } 1110 }
1100 _numProcesses++; 1111 _numProcesses++;
1101 } 1112 }
1102 } 1113 }
1103 } 1114 }
OLDNEW
« no previous file with comments | « tools/testing/dart/test_progress.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698