OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 478 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
489 * its [TestCase], starting the test process (and first, a compilation | 489 * its [TestCase], starting the test process (and first, a compilation |
490 * process if the TestCase is a [BrowserTestCase]), creating a timeout | 490 * process if the TestCase is a [BrowserTestCase]), creating a timeout |
491 * timer, and recording the results in a new [TestOutput] object, which it | 491 * timer, and recording the results in a new [TestOutput] object, which it |
492 * attaches to the TestCase. The lifetime of the RunningProcess is limited | 492 * attaches to the TestCase. The lifetime of the RunningProcess is limited |
493 * to the time it takes to start the process, run the process, and record | 493 * to the time it takes to start the process, run the process, and record |
494 * the result; there are no pointers to it, so it should be available to | 494 * the result; there are no pointers to it, so it should be available to |
495 * be garbage collected as soon as it is done. | 495 * be garbage collected as soon as it is done. |
496 */ | 496 */ |
497 class RunningProcess { | 497 class RunningProcess { |
498 ProcessQueue processQueue; | 498 ProcessQueue processQueue; |
499 Process process; | 499 InteractiveProcess process; |
500 TestCase testCase; | 500 TestCase testCase; |
501 bool timedOut = false; | 501 bool timedOut = false; |
502 Date startTime; | 502 Date startTime; |
503 Timer timeoutTimer; | 503 Timer timeoutTimer; |
504 List<String> stdout; | 504 List<String> stdout; |
505 List<String> stderr; | 505 List<String> stderr; |
506 List<Function> handlers; | 506 List<Function> handlers; |
507 bool allowRetries; | 507 bool allowRetries; |
508 | 508 |
509 /** Which command of [testCase.commands] is currently being executed. */ | 509 /** Which command of [testCase.commands] is currently being executed. */ |
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
597 } | 597 } |
598 | 598 |
599 void runCommand(Command command, | 599 void runCommand(Command command, |
600 void exitHandler(int exitCode)) { | 600 void exitHandler(int exitCode)) { |
601 if (Platform.operatingSystem == 'windows') { | 601 if (Platform.operatingSystem == 'windows') { |
602 // Windows can't handle the first command if it is a .bat file or the like | 602 // Windows can't handle the first command if it is a .bat file or the like |
603 // with the slashes going the other direction. | 603 // with the slashes going the other direction. |
604 // TODO(efortuna): Remove this when fixed (Issue 1306). | 604 // TODO(efortuna): Remove this when fixed (Issue 1306). |
605 command.executable = command.executable.replaceAll('/', '\\'); | 605 command.executable = command.executable.replaceAll('/', '\\'); |
606 } | 606 } |
607 process = new Process.start(command.executable, command.arguments); | 607 process = Process.start(command.executable, command.arguments); |
608 process.onExit = exitHandler; | 608 process.onExit = exitHandler; |
609 process.onError = (e) { | 609 process.onError = (e) { |
610 print("Error starting process:"); | 610 print("Error starting process:"); |
611 print(" Command: $command"); | 611 print(" Command: $command"); |
612 print(" Error: $e"); | 612 print(" Error: $e"); |
613 }; | 613 }; |
614 startTime = new Date.now(); | 614 startTime = new Date.now(); |
615 InputStream stdoutStream = process.stdout; | 615 InputStream stdoutStream = process.stdout; |
616 InputStream stderrStream = process.stderr; | 616 InputStream stderrStream = process.stderr; |
617 StringInputStream stdoutStringStream = new StringInputStream(stdoutStream); | 617 StringInputStream stdoutStringStream = new StringInputStream(stdoutStream); |
(...skipping 11 matching lines...) Expand all Loading... |
629 void timeoutHandler(Timer unusedTimer) { | 629 void timeoutHandler(Timer unusedTimer) { |
630 timedOut = true; | 630 timedOut = true; |
631 process.kill(); | 631 process.kill(); |
632 } | 632 } |
633 } | 633 } |
634 | 634 |
635 class BatchRunnerProcess { | 635 class BatchRunnerProcess { |
636 String _executable; | 636 String _executable; |
637 List<String> _batchArguments; | 637 List<String> _batchArguments; |
638 | 638 |
639 Process _process; | 639 InteractiveProcess _process; |
640 StringInputStream _stdoutStream; | 640 StringInputStream _stdoutStream; |
641 StringInputStream _stderrStream; | 641 StringInputStream _stderrStream; |
642 | 642 |
643 TestCase _currentTest; | 643 TestCase _currentTest; |
644 List<String> _testStdout; | 644 List<String> _testStdout; |
645 List<String> _testStderr; | 645 List<String> _testStderr; |
646 bool _stdoutDrained = false; | 646 bool _stdoutDrained = false; |
647 bool _stderrDrained = false; | 647 bool _stderrDrained = false; |
648 Date _startTime; | 648 Date _startTime; |
649 Timer _timer; | 649 Timer _timer; |
(...skipping 151 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
801 _process.onExit = (exitCode) { | 801 _process.onExit = (exitCode) { |
802 _process.close(); | 802 _process.close(); |
803 _startProcess(() { | 803 _startProcess(() { |
804 _reportResult(">>> TEST TIMEOUT"); | 804 _reportResult(">>> TEST TIMEOUT"); |
805 }); | 805 }); |
806 }; | 806 }; |
807 _process.kill(); | 807 _process.kill(); |
808 } | 808 } |
809 | 809 |
810 void _startProcess(then) { | 810 void _startProcess(then) { |
811 _process = new Process.start(_executable, _batchArguments); | 811 _process = Process.start(_executable, _batchArguments); |
812 _stdoutStream = new StringInputStream(_process.stdout); | 812 _stdoutStream = new StringInputStream(_process.stdout); |
813 _stderrStream = new StringInputStream(_process.stderr); | 813 _stderrStream = new StringInputStream(_process.stderr); |
814 _testStdout = new List<String>(); | 814 _testStdout = new List<String>(); |
815 _testStderr = new List<String>(); | 815 _testStderr = new List<String>(); |
816 _stdoutDrained = false; | 816 _stdoutDrained = false; |
817 _stderrDrained = false; | 817 _stderrDrained = false; |
818 _stdoutStream.onLine = _readStdout(_stdoutStream, _testStdout); | 818 _stdoutStream.onLine = _readStdout(_stdoutStream, _testStdout); |
819 _stderrStream.onLine = _readStderr(_stderrStream, _testStderr); | 819 _stderrStream.onLine = _readStderr(_stderrStream, _testStderr); |
820 _process.onExit = _exitHandler; | 820 _process.onExit = _exitHandler; |
821 _process.onError = (e) { | 821 _process.onError = (e) { |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
867 /** | 867 /** |
868 * String indicating the browser used to run the tests. Empty if no browser | 868 * String indicating the browser used to run the tests. Empty if no browser |
869 * used. | 869 * used. |
870 */ | 870 */ |
871 String browserUsed = ''; | 871 String browserUsed = ''; |
872 | 872 |
873 /** | 873 /** |
874 * Process running the selenium server .jar (only used for Safari and Opera | 874 * Process running the selenium server .jar (only used for Safari and Opera |
875 * tests.) | 875 * tests.) |
876 */ | 876 */ |
877 Process _seleniumServer = null; | 877 InteractiveProcess _seleniumServer = null; |
878 | 878 |
879 /** True if we are in the process of starting the server. */ | 879 /** True if we are in the process of starting the server. */ |
880 bool _startingServer = false; | 880 bool _startingServer = false; |
881 | 881 |
882 /** True if we find that there is already a selenium jar running. */ | 882 /** True if we find that there is already a selenium jar running. */ |
883 bool _seleniumAlreadyRunning = false; | 883 bool _seleniumAlreadyRunning = false; |
884 | 884 |
885 ProcessQueue(int this._maxProcesses, | 885 ProcessQueue(int this._maxProcesses, |
886 String progress, | 886 String progress, |
887 Date startTime, | 887 Date startTime, |
(...skipping 28 matching lines...) Expand all Loading... |
916 _checkDone(); | 916 _checkDone(); |
917 } | 917 } |
918 | 918 |
919 String globalTemporaryDirectory() { | 919 String globalTemporaryDirectory() { |
920 if (_temporaryDirectory != null) return _temporaryDirectory; | 920 if (_temporaryDirectory != null) return _temporaryDirectory; |
921 | 921 |
922 if (Platform.operatingSystem == 'windows') { | 922 if (Platform.operatingSystem == 'windows') { |
923 throw new Exception( | 923 throw new Exception( |
924 'Test suite requires temporary directory. Not supported on Windows.'); | 924 'Test suite requires temporary directory. Not supported on Windows.'); |
925 } | 925 } |
926 var tempDir = new Directory(''); | 926 var tempDir = new Directory('').createTempSync(); |
927 tempDir.createTempSync(); | |
928 _temporaryDirectory = tempDir.path; | 927 _temporaryDirectory = tempDir.path; |
929 return _temporaryDirectory; | 928 return _temporaryDirectory; |
930 } | 929 } |
931 | 930 |
932 /** | 931 /** |
933 * Perform any cleanup needed once all tests in a TestSuite have completed | 932 * Perform any cleanup needed once all tests in a TestSuite have completed |
934 * and notify our progress indicator that we are done. | 933 * and notify our progress indicator that we are done. |
935 */ | 934 */ |
936 void _cleanupAndMarkDone() { | 935 void _cleanupAndMarkDone() { |
937 if (browserUsed != '' && _seleniumServer != null) { | 936 if (browserUsed != '' && _seleniumServer != null) { |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
992 if (!_isSeleniumAvailable && !_startingServer) { | 991 if (!_isSeleniumAvailable && !_startingServer) { |
993 _startingServer = true; | 992 _startingServer = true; |
994 | 993 |
995 // Check to see if the jar was already running before the program started. | 994 // Check to see if the jar was already running before the program started. |
996 String cmd = 'ps'; | 995 String cmd = 'ps'; |
997 var arg = ['aux']; | 996 var arg = ['aux']; |
998 if (Platform.operatingSystem == 'windows') { | 997 if (Platform.operatingSystem == 'windows') { |
999 cmd = 'tasklist'; | 998 cmd = 'tasklist'; |
1000 arg.add('/v'); | 999 arg.add('/v'); |
1001 } | 1000 } |
1002 Process p = new Process.start(cmd, arg); | 1001 InteractiveProcess p = Process.start(cmd, arg); |
1003 final StringInputStream stdoutStringStream = | 1002 final StringInputStream stdoutStringStream = |
1004 new StringInputStream(p.stdout); | 1003 new StringInputStream(p.stdout); |
1005 p.onError = (e) { | 1004 p.onError = (e) { |
1006 print("Error starting process:"); | 1005 print("Error starting process:"); |
1007 print(" Command: $cmd ${Strings.join(arg, ' ')}"); | 1006 print(" Command: $cmd ${Strings.join(arg, ' ')}"); |
1008 print(" Error: $e"); | 1007 print(" Error: $e"); |
1009 }; | 1008 }; |
1010 stdoutStringStream.onLine = () { | 1009 stdoutStringStream.onLine = () { |
1011 var line = stdoutStringStream.readLine(); | 1010 var line = stdoutStringStream.readLine(); |
1012 while (null != line) { | 1011 while (null != line) { |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1056 /** | 1055 /** |
1057 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 | 1056 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 |
1058 * Java server. | 1057 * Java server. |
1059 */ | 1058 */ |
1060 void _startSeleniumServer() { | 1059 void _startSeleniumServer() { |
1061 // Get the absolute path to the Selenium jar. | 1060 // Get the absolute path to the Selenium jar. |
1062 String filePath = new Options().script; | 1061 String filePath = new Options().script; |
1063 String pathSep = Platform.pathSeparator; | 1062 String pathSep = Platform.pathSeparator; |
1064 int index = filePath.lastIndexOf(pathSep); | 1063 int index = filePath.lastIndexOf(pathSep); |
1065 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; | 1064 filePath = filePath.substring(0, index) + '${pathSep}testing${pathSep}'; |
1066 var dir = new Directory(filePath); | 1065 var lister = new Directory(filePath).list(); |
1067 dir.onFile = (String file) { | 1066 lister.onFile = (String file) { |
1068 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) | 1067 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) |
1069 && _seleniumServer == null) { | 1068 && _seleniumServer == null) { |
1070 _seleniumServer = new Process.start('java', ['-jar', file]); | 1069 _seleniumServer = Process.start('java', ['-jar', file]); |
1071 _seleniumServer.onError = (e) { | 1070 _seleniumServer.onError = (e) { |
1072 print("Error starting process:"); | 1071 print("Error starting process:"); |
1073 print(" Command: java -jar $file"); | 1072 print(" Command: java -jar $file"); |
1074 print(" Error: $e"); | 1073 print(" Error: $e"); |
1075 }; | 1074 }; |
1076 // Heads up: there seems to an obscure data race of some form in | 1075 // Heads up: there seems to an obscure data race of some form in |
1077 // the VM between launching the server process and launching the test | 1076 // the VM between launching the server process and launching the test |
1078 // tasks that disappears when you read IO (which is convenient, since | 1077 // tasks that disappears when you read IO (which is convenient, since |
1079 // that is our condition for knowing that the server is ready). | 1078 // that is our condition for knowing that the server is ready). |
1080 StringInputStream stdoutStringStream = | 1079 StringInputStream stdoutStringStream = |
1081 new StringInputStream(_seleniumServer.stdout); | 1080 new StringInputStream(_seleniumServer.stdout); |
1082 StringInputStream stderrStringStream = | 1081 StringInputStream stderrStringStream = |
1083 new StringInputStream(_seleniumServer.stderr); | 1082 new StringInputStream(_seleniumServer.stderr); |
1084 stdoutStringStream.onLine = | 1083 stdoutStringStream.onLine = |
1085 makeSeleniumServerHandler(stdoutStringStream); | 1084 makeSeleniumServerHandler(stdoutStringStream); |
1086 stderrStringStream.onLine = | 1085 stderrStringStream.onLine = |
1087 makeSeleniumServerHandler(stderrStringStream); | 1086 makeSeleniumServerHandler(stderrStringStream); |
1088 } | 1087 } |
1089 }; | 1088 }; |
1090 dir.list(); | |
1091 } | 1089 } |
1092 | 1090 |
1093 void _terminateBatchRunners() { | 1091 void _terminateBatchRunners() { |
1094 for (var runners in _batchProcesses.getValues()) { | 1092 for (var runners in _batchProcesses.getValues()) { |
1095 for (var runner in runners) { | 1093 for (var runner in runners) { |
1096 runner.terminate(); | 1094 runner.terminate(); |
1097 } | 1095 } |
1098 } | 1096 } |
1099 } | 1097 } |
1100 | 1098 |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1162 // the developer doesn't waste his or her time trying to fix a bunch of | 1160 // the developer doesn't waste his or her time trying to fix a bunch of |
1163 // tests that appear to be broken but were actually just flakes that | 1161 // tests that appear to be broken but were actually just flakes that |
1164 // didn't get retried because there had already been one failure. | 1162 // didn't get retried because there had already been one failure. |
1165 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1163 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
1166 new RunningProcess(test, allowRetry, this).start(); | 1164 new RunningProcess(test, allowRetry, this).start(); |
1167 } | 1165 } |
1168 _numProcesses++; | 1166 _numProcesses++; |
1169 } | 1167 } |
1170 } | 1168 } |
1171 } | 1169 } |
OLD | NEW |