| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| (...skipping 236 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 247 return new TestOutputImpl(testCase, exitCode, timedOut, | 247 return new TestOutputImpl(testCase, exitCode, timedOut, |
| 248 stdout, stderr, time); | 248 stdout, stderr, time); |
| 249 } | 249 } |
| 250 | 250 |
| 251 String get result() => | 251 String get result() => |
| 252 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); | 252 hasCrashed ? CRASH : (hasTimedOut ? TIMEOUT : (hasFailed ? FAIL : PASS)); |
| 253 | 253 |
| 254 bool get unexpectedOutput() => !testCase.expectedOutcomes.contains(result); | 254 bool get unexpectedOutput() => !testCase.expectedOutcomes.contains(result); |
| 255 | 255 |
| 256 bool get hasCrashed() { | 256 bool get hasCrashed() { |
| 257 if (new Platform().operatingSystem() == 'windows') { | 257 if (Platform.operatingSystem() == 'windows') { |
| 258 if (exitCode != 0) { | 258 if (exitCode != 0) { |
| 259 // Suppress some flaky errors that crash the VM. | 259 // Suppress some flaky errors that crash the VM. |
| 260 // TODO(sigmund,efortuna): remove this when bug 2124 gets fixed. | 260 // TODO(sigmund,efortuna): remove this when bug 2124 gets fixed. |
| 261 for (String line in testCase.output.stdout) { | 261 for (String line in testCase.output.stdout) { |
| 262 if (line.startsWith('Kind:')) { | 262 if (line.startsWith('Kind:')) { |
| 263 if (!alreadyPrintedWarning) { | 263 if (!alreadyPrintedWarning) { |
| 264 print("WARNING: VM crashed: $line, exit code: $exitCode. " | 264 print("WARNING: VM crashed: $line, exit code: $exitCode. " |
| 265 " This is a fake pass!!"); | 265 " This is a fake pass!!"); |
| 266 alreadyPrintedWarning = true; | 266 alreadyPrintedWarning = true; |
| 267 } | 267 } |
| (...skipping 17 matching lines...) Expand all Loading... |
| 285 bool get hasTimedOut() => timedOut; | 285 bool get hasTimedOut() => timedOut; |
| 286 | 286 |
| 287 bool get didFail() { | 287 bool get didFail() { |
| 288 return (exitCode != 0 && !hasCrashed); | 288 return (exitCode != 0 && !hasCrashed); |
| 289 } | 289 } |
| 290 | 290 |
| 291 // Reverse result of a negative test. | 291 // Reverse result of a negative test. |
| 292 bool get hasFailed() { | 292 bool get hasFailed() { |
| 293 // TODO(efortuna): This is a total hack to keep our buildbots (more) green | 293 // TODO(efortuna): This is a total hack to keep our buildbots (more) green |
| 294 // while the VM team solves Issue 2124. Remove when issue is fixed. | 294 // while the VM team solves Issue 2124. Remove when issue is fixed. |
| 295 if (new Platform().operatingSystem() == 'windows' && (exitCode == 253 || | 295 if (Platform.operatingSystem() == 'windows' && (exitCode == 253 || |
| 296 exitCode == 3)) { | 296 exitCode == 3)) { |
| 297 for (String line in testCase.output.stdout) { | 297 for (String line in testCase.output.stdout) { |
| 298 if (line.startsWith('VM exited with signal 1073741819') || | 298 if (line.startsWith('VM exited with signal 1073741819') || |
| 299 line.startsWith('Kind:')) { | 299 line.startsWith('Kind:')) { |
| 300 if (!alreadyPrintedWarning) { | 300 if (!alreadyPrintedWarning) { |
| 301 print("WARNING: VM crashed: $line This is a fake pass!!"); | 301 print("WARNING: VM crashed: $line This is a fake pass!!"); |
| 302 alreadyPrintedWarning = true; | 302 alreadyPrintedWarning = true; |
| 303 } | 303 } |
| 304 return testCase.expectedOutcomes.iterator().next() == FAIL; | 304 return testCase.expectedOutcomes.iterator().next() == FAIL; |
| 305 } | 305 } |
| (...skipping 307 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 613 void start() { | 613 void start() { |
| 614 Expect.isFalse(testCase.expectedOutcomes.contains(SKIP)); | 614 Expect.isFalse(testCase.expectedOutcomes.contains(SKIP)); |
| 615 stdout = new List<String>(); | 615 stdout = new List<String>(); |
| 616 stderr = new List<String>(); | 616 stderr = new List<String>(); |
| 617 currentStep = 0; | 617 currentStep = 0; |
| 618 runCommand(testCase.commands[currentStep++], stepExitHandler); | 618 runCommand(testCase.commands[currentStep++], stepExitHandler); |
| 619 } | 619 } |
| 620 | 620 |
| 621 void runCommand(Command command, | 621 void runCommand(Command command, |
| 622 void exitHandler(int exitCode)) { | 622 void exitHandler(int exitCode)) { |
| 623 if (new Platform().operatingSystem() == 'windows') { | 623 if (Platform.operatingSystem() == 'windows') { |
| 624 // Windows can't handle the first command if it is a .bat file or the like | 624 // Windows can't handle the first command if it is a .bat file or the like |
| 625 // with the slashes going the other direction. | 625 // with the slashes going the other direction. |
| 626 // TODO(efortuna): Remove this when fixed (Issue 1306). | 626 // TODO(efortuna): Remove this when fixed (Issue 1306). |
| 627 command.executable = command.executable.replaceAll('/', '\\'); | 627 command.executable = command.executable.replaceAll('/', '\\'); |
| 628 } | 628 } |
| 629 process = new Process.start(command.executable, command.arguments); | 629 process = new Process.start(command.executable, command.arguments); |
| 630 process.onExit = exitHandler; | 630 process.onExit = exitHandler; |
| 631 startTime = new Date.now(); | 631 startTime = new Date.now(); |
| 632 InputStream stdoutStream = process.stdout; | 632 InputStream stdoutStream = process.stdout; |
| 633 InputStream stderrStream = process.stderr; | 633 InputStream stderrStream = process.stderr; |
| (...skipping 287 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 921 } | 921 } |
| 922 | 922 |
| 923 void _testListerDone() { | 923 void _testListerDone() { |
| 924 _activeTestListers--; | 924 _activeTestListers--; |
| 925 _checkDone(); | 925 _checkDone(); |
| 926 } | 926 } |
| 927 | 927 |
| 928 String globalTemporaryDirectory() { | 928 String globalTemporaryDirectory() { |
| 929 if (_temporaryDirectory != null) return _temporaryDirectory; | 929 if (_temporaryDirectory != null) return _temporaryDirectory; |
| 930 | 930 |
| 931 if (new Platform().operatingSystem() == 'windows') { | 931 if (Platform.operatingSystem() == 'windows') { |
| 932 throw new Exception( | 932 throw new Exception( |
| 933 'Test suite requires temporary directory. Not supported on Windows.'); | 933 'Test suite requires temporary directory. Not supported on Windows.'); |
| 934 } | 934 } |
| 935 var tempDir = new Directory(''); | 935 var tempDir = new Directory(''); |
| 936 tempDir.createTempSync(); | 936 tempDir.createTempSync(); |
| 937 _temporaryDirectory = tempDir.path; | 937 _temporaryDirectory = tempDir.path; |
| 938 return _temporaryDirectory; | 938 return _temporaryDirectory; |
| 939 } | 939 } |
| 940 | 940 |
| 941 /** | 941 /** |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 974 }; | 974 }; |
| 975 } | 975 } |
| 976 } | 976 } |
| 977 } | 977 } |
| 978 } | 978 } |
| 979 | 979 |
| 980 /** | 980 /** |
| 981 * True if we are using a browser + platform combination that needs the | 981 * True if we are using a browser + platform combination that needs the |
| 982 * Selenium server jar. | 982 * Selenium server jar. |
| 983 */ | 983 */ |
| 984 bool get _needsSelenium() => new Platform().operatingSystem() == 'macos' && | 984 bool get _needsSelenium() => Platform.operatingSystem() == 'macos' && |
| 985 browserUsed == 'safari'; | 985 browserUsed == 'safari'; |
| 986 | 986 |
| 987 /** True if the Selenium Server is ready to be used. */ | 987 /** True if the Selenium Server is ready to be used. */ |
| 988 bool get _isSeleniumAvailable() => _seleniumServer != null || | 988 bool get _isSeleniumAvailable() => _seleniumServer != null || |
| 989 _seleniumAlreadyRunning; | 989 _seleniumAlreadyRunning; |
| 990 | 990 |
| 991 /** | 991 /** |
| 992 * Restart all the processes that have been waiting/stopped for the server to | 992 * Restart all the processes that have been waiting/stopped for the server to |
| 993 * start up. If we just call this once we end up with a single-"threaded" run. | 993 * start up. If we just call this once we end up with a single-"threaded" run. |
| 994 */ | 994 */ |
| 995 void resumeTesting() { | 995 void resumeTesting() { |
| 996 for (int i = 0; i < _maxProcesses; i++) _tryRunTest(); | 996 for (int i = 0; i < _maxProcesses; i++) _tryRunTest(); |
| 997 } | 997 } |
| 998 | 998 |
| 999 /** Start the Selenium Server jar, if appropriate for this platform. */ | 999 /** Start the Selenium Server jar, if appropriate for this platform. */ |
| 1000 void _ensureSeleniumServerRunning() { | 1000 void _ensureSeleniumServerRunning() { |
| 1001 if (!_isSeleniumAvailable && !_startingServer) { | 1001 if (!_isSeleniumAvailable && !_startingServer) { |
| 1002 _startingServer = true; | 1002 _startingServer = true; |
| 1003 | 1003 |
| 1004 // Check to see if the jar was already running before the program started. | 1004 // Check to see if the jar was already running before the program started. |
| 1005 String cmd = 'ps'; | 1005 String cmd = 'ps'; |
| 1006 var arg = ['aux']; | 1006 var arg = ['aux']; |
| 1007 if (new Platform().operatingSystem() == 'windows') { | 1007 if (Platform.operatingSystem() == 'windows') { |
| 1008 cmd = 'tasklist'; | 1008 cmd = 'tasklist'; |
| 1009 arg.add('/v'); | 1009 arg.add('/v'); |
| 1010 } | 1010 } |
| 1011 Process p = new Process.start(cmd, arg); | 1011 Process p = new Process.start(cmd, arg); |
| 1012 final StringInputStream stdoutStringStream = | 1012 final StringInputStream stdoutStringStream = |
| 1013 new StringInputStream(p.stdout); | 1013 new StringInputStream(p.stdout); |
| 1014 stdoutStringStream.onLine = () { | 1014 stdoutStringStream.onLine = () { |
| 1015 var line = stdoutStringStream.readLine(); | 1015 var line = stdoutStringStream.readLine(); |
| 1016 while (null != line) { | 1016 while (null != line) { |
| 1017 if (const RegExp(@".*selenium-server-standalone.*").hasMatch(line)) { | 1017 if (const RegExp(@".*selenium-server-standalone.*").hasMatch(line)) { |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1057 }; | 1057 }; |
| 1058 } | 1058 } |
| 1059 | 1059 |
| 1060 /** | 1060 /** |
| 1061 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 | 1061 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 |
| 1062 * Java server. | 1062 * Java server. |
| 1063 */ | 1063 */ |
| 1064 void _startSeleniumServer() { | 1064 void _startSeleniumServer() { |
| 1065 // Get the absolute path to the Selenium jar. | 1065 // Get the absolute path to the Selenium jar. |
| 1066 String filePath = new Options().script; | 1066 String filePath = new Options().script; |
| 1067 String pathSep = new Platform().pathSeparator(); | 1067 String pathSep = Platform.pathSeparator(); |
| 1068 int index = filePath.lastIndexOf(pathSep); | 1068 int index = filePath.lastIndexOf(pathSep); |
| 1069 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; | 1069 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; |
| 1070 var dir = new Directory(filePath); | 1070 var dir = new Directory(filePath); |
| 1071 dir.onFile = (String file) { | 1071 dir.onFile = (String file) { |
| 1072 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) | 1072 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) |
| 1073 && _seleniumServer == null) { | 1073 && _seleniumServer == null) { |
| 1074 _seleniumServer = new Process.start('java', ['-jar', file]); | 1074 _seleniumServer = new Process.start('java', ['-jar', file]); |
| 1075 // Heads up: there seems to an obscure data race of some form in | 1075 // Heads up: there seems to an obscure data race of some form in |
| 1076 // the VM between launching the server process and launching the test | 1076 // the VM between launching the server process and launching the test |
| 1077 // tasks that disappears when you read IO (which is convenient, since | 1077 // tasks that disappears when you read IO (which is convenient, since |
| (...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1160 // the developer doesn't waste his or her time trying to fix a bunch of | 1160 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1161 // tests that appear to be broken but were actually just flakes that | 1161 // tests that appear to be broken but were actually just flakes that |
| 1162 // didn't get retried because there had already been one failure. | 1162 // didn't get retried because there had already been one failure. |
| 1163 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1163 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1164 new RunningProcess(test, allowRetry, this).start(); | 1164 new RunningProcess(test, allowRetry, this).start(); |
| 1165 } | 1165 } |
| 1166 _numProcesses++; | 1166 _numProcesses++; |
| 1167 } | 1167 } |
| 1168 } | 1168 } |
| 1169 } | 1169 } |
| OLD | NEW |