| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
| 7 * | 7 * |
| 8 * This library includes: | 8 * This library includes: |
| 9 * | 9 * |
| 10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
| (...skipping 28 matching lines...) Expand all Loading... |
| 39 * Most TestSuites represent a directory or directory tree containing tests, | 39 * Most TestSuites represent a directory or directory tree containing tests, |
| 40 * and a status file containing the expected results when these tests are run. | 40 * and a status file containing the expected results when these tests are run. |
| 41 */ | 41 */ |
| 42 interface TestSuite { | 42 interface TestSuite { |
| 43 /** | 43 /** |
| 44 * Call the callback function onTest with a [TestCase] argument for each | 44 * Call the callback function onTest with a [TestCase] argument for each |
| 45 * test in the suite. When all tests have been processed, call [onDone]. | 45 * test in the suite. When all tests have been processed, call [onDone]. |
| 46 * | 46 * |
| 47 * The [testCache] argument provides a persistent store that can be used to | 47 * The [testCache] argument provides a persistent store that can be used to |
| 48 * cache information about the test suite, so that directories do not need | 48 * cache information about the test suite, so that directories do not need |
| 49 * to be listed each time. If the tests require a temporary directory for | 49 * to be listed each time. |
| 50 * their files, they can get one from [globalTempDir]. | |
| 51 */ | 50 */ |
| 52 void forEachTest(Function onTest, Map testCache, String globalTempDir(), | 51 void forEachTest(Function onTest, Map testCache, [Function onDone]); |
| 53 [Function onDone]); | |
| 54 } | 52 } |
| 55 | 53 |
| 56 | 54 |
| 57 class CCTestListerIsolate extends Isolate { | 55 class CCTestListerIsolate extends Isolate { |
| 58 CCTestListerIsolate() : super.heavy(); | 56 CCTestListerIsolate() : super.heavy(); |
| 59 | 57 |
| 60 void main() { | 58 void main() { |
| 61 port.receive((String runnerPath, SendPort replyTo) { | 59 port.receive((String runnerPath, SendPort replyTo) { |
| 62 var p = Process.start(runnerPath, ["--list"]); | 60 var p = Process.start(runnerPath, ["--list"]); |
| 63 StringInputStream stdoutStream = new StringInputStream(p.stdout); | 61 StringInputStream stdoutStream = new StringInputStream(p.stdout); |
| (...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 144 args.addAll(TestUtils.standardOptions(configuration)); | 142 args.addAll(TestUtils.standardOptions(configuration)); |
| 145 | 143 |
| 146 doTest(new TestCase(constructedName, | 144 doTest(new TestCase(constructedName, |
| 147 [new Command(runnerPath, args)], | 145 [new Command(runnerPath, args)], |
| 148 configuration, | 146 configuration, |
| 149 completeHandler, | 147 completeHandler, |
| 150 expectations)); | 148 expectations)); |
| 151 } | 149 } |
| 152 } | 150 } |
| 153 | 151 |
| 154 void forEachTest(Function onTest, Map testCache, String globalTempDir(), | 152 void forEachTest(Function onTest, Map testCache, [Function onDone]) { |
| 155 [Function onDone]) { | |
| 156 doTest = onTest; | 153 doTest = onTest; |
| 157 doDone = (ignore) => (onDone != null) ? onDone() : null; | 154 doDone = (ignore) => (onDone != null) ? onDone() : null; |
| 158 | 155 |
| 159 var filesRead = 0; | 156 var filesRead = 0; |
| 160 void statusFileRead() { | 157 void statusFileRead() { |
| 161 filesRead++; | 158 filesRead++; |
| 162 if (filesRead == statusFilePaths.length) { | 159 if (filesRead == statusFilePaths.length) { |
| 163 receiveTestName = new ReceivePort(); | 160 receiveTestName = new ReceivePort(); |
| 164 new CCTestListerIsolate().spawn().then((port) { | 161 new CCTestListerIsolate().spawn().then((port) { |
| 165 port.send(runnerPath, receiveTestName.toSendPort()); | 162 port.send(runnerPath, receiveTestName.toSendPort()); |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 206 String suiteName; | 203 String suiteName; |
| 207 String directoryPath; | 204 String directoryPath; |
| 208 List<String> statusFilePaths; | 205 List<String> statusFilePaths; |
| 209 Function doTest; | 206 Function doTest; |
| 210 Function doDone; | 207 Function doDone; |
| 211 int activeTestGenerators = 0; | 208 int activeTestGenerators = 0; |
| 212 bool listingDone = false; | 209 bool listingDone = false; |
| 213 TestExpectations testExpectations; | 210 TestExpectations testExpectations; |
| 214 List<TestInformation> cachedTests; | 211 List<TestInformation> cachedTests; |
| 215 final String dartDir; | 212 final String dartDir; |
| 216 Function globalTemporaryDirectory; | |
| 217 Predicate<String> isTestFilePredicate; | 213 Predicate<String> isTestFilePredicate; |
| 218 bool _listRecursive; | 214 bool _listRecursive; |
| 219 | 215 |
| 220 StandardTestSuite(Map this.configuration, | 216 StandardTestSuite(Map this.configuration, |
| 221 String this.suiteName, | 217 String this.suiteName, |
| 222 String this.directoryPath, | 218 String this.directoryPath, |
| 223 List<String> this.statusFilePaths, | 219 List<String> this.statusFilePaths, |
| 224 [Predicate<String> this.isTestFilePredicate, | 220 [Predicate<String> this.isTestFilePredicate, |
| 225 bool recursive = false]) | 221 bool recursive = false]) |
| 226 : dartDir = TestUtils.dartDir(), _listRecursive = recursive; | 222 : dartDir = TestUtils.dartDir(), _listRecursive = recursive; |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 273 | 269 |
| 274 return filename.endsWith("Test.dart"); | 270 return filename.endsWith("Test.dart"); |
| 275 } | 271 } |
| 276 | 272 |
| 277 bool listRecursively() => _listRecursive; | 273 bool listRecursively() => _listRecursive; |
| 278 | 274 |
| 279 String shellPath() => TestUtils.dartShellFileName(configuration); | 275 String shellPath() => TestUtils.dartShellFileName(configuration); |
| 280 | 276 |
| 281 List<String> additionalOptions(String filename) => []; | 277 List<String> additionalOptions(String filename) => []; |
| 282 | 278 |
| 283 void forEachTest(Function onTest, Map testCache, String globalTempDir(), | 279 void forEachTest(Function onTest, Map testCache, [Function onDone = null]) { |
| 284 [Function onDone = null]) { | |
| 285 // If DumpRenderTree/Dartium is required, and not yet updated, | 280 // If DumpRenderTree/Dartium is required, and not yet updated, |
| 286 // wait for update. | 281 // wait for update. |
| 287 var updater = runtimeUpdater(configuration); | 282 var updater = runtimeUpdater(configuration); |
| 288 if (updater !== null && !updater.updated) { | 283 if (updater !== null && !updater.updated) { |
| 289 Expect.isTrue(updater.isActive); | 284 Expect.isTrue(updater.isActive); |
| 290 updater.onUpdated.add(() { | 285 updater.onUpdated.add(() { |
| 291 forEachTest(onTest, testCache, globalTempDir, onDone); | 286 forEachTest(onTest, testCache, onDone); |
| 292 }); | 287 }); |
| 293 return; | 288 return; |
| 294 } | 289 } |
| 295 | 290 |
| 296 doTest = onTest; | 291 doTest = onTest; |
| 297 doDone = (onDone != null) ? onDone : (() => null); | 292 doDone = (onDone != null) ? onDone : (() => null); |
| 298 globalTemporaryDirectory = globalTempDir; | |
| 299 | 293 |
| 300 var filesRead = 0; | 294 var filesRead = 0; |
| 301 void statusFileRead() { | 295 void statusFileRead() { |
| 302 filesRead++; | 296 filesRead++; |
| 303 if (filesRead == statusFilePaths.length) { | 297 if (filesRead == statusFilePaths.length) { |
| 304 // Checked if we have already found and generated the tests for | 298 // Checked if we have already found and generated the tests for |
| 305 // this suite. | 299 // this suite. |
| 306 if (!testCache.containsKey(suiteName)) { | 300 if (!testCache.containsKey(suiteName)) { |
| 307 cachedTests = testCache[suiteName] = []; | 301 cachedTests = testCache[suiteName] = []; |
| 308 processDirectory(); | 302 processDirectory(); |
| (...skipping 399 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 708 Expect.fail('unimplemented compiler $compiler'); | 702 Expect.fail('unimplemented compiler $compiler'); |
| 709 } | 703 } |
| 710 if (executable.endsWith('.dart')) { | 704 if (executable.endsWith('.dart')) { |
| 711 // Run the compiler script via the Dart VM. | 705 // Run the compiler script via the Dart VM. |
| 712 args.insertRange(0, 1, executable); | 706 args.insertRange(0, 1, executable); |
| 713 executable = TestUtils.dartShellFileName(configuration); | 707 executable = TestUtils.dartShellFileName(configuration); |
| 714 } | 708 } |
| 715 return new Command(executable, args); | 709 return new Command(executable, args); |
| 716 } | 710 } |
| 717 | 711 |
| 718 bool get requiresCleanTemporaryDirectory() => | |
| 719 configuration['compiler'] == 'dartc'; | |
| 720 | |
| 721 /** | 712 /** |
| 722 * Create a directory for the generated test. If a Dart language test | 713 * Create a directory for the generated test. If a Dart language test |
| 723 * needs to be run in a browser, the Dart test needs to be embedded in | 714 * needs to be run in a browser, the Dart test needs to be embedded in |
| 724 * an HTML page, with a testing framework based on scripting and DOM events. | 715 * an HTML page, with a testing framework based on scripting and DOM events. |
| 725 * These scripts and pages are written to a generated_test directory, | 716 * These scripts and pages are written to a generated_test directory |
| 726 * usually inside the build directory of the checkout. | 717 * inside the build directory of the checkout. |
| 727 * | |
| 728 * Some tests, such as those using the dartc compiler, need to be run | |
| 729 * with an empty directory as the compiler's work directory. These | |
| 730 * tests are copied to a subdirectory of a system-provided temporary | |
| 731 * directory, which is deleted at the end of the test run unless the | |
| 732 * --keep-temporary-files flag is given. | |
| 733 * | 718 * |
| 734 * Those tests which are already HTML web applications (web tests), with | 719 * Those tests which are already HTML web applications (web tests), with |
| 735 * resources including CSS files and HTML files, need to be compiled into | 720 * resources including CSS files and HTML files, need to be compiled into |
| 736 * a work directory where the relative URLS to the resources work. | 721 * a work directory where the relative URLS to the resources work. |
| 737 * We use a subdirectory of the build directory that is the same number | 722 * We use a subdirectory of the build directory that is the same number |
| 738 * of levels down in the checkout as the original path of the web test. | 723 * of levels down in the checkout as the original path of the web test. |
| 739 */ | 724 */ |
| 740 Directory createOutputDirectory(String testPath, String optionsName) { | 725 Directory createOutputDirectory(String testPath, String optionsName) { |
| 741 String testUniqueName = | 726 String testUniqueName = |
| 742 testPath.substring(dartDir.length + 1, testPath.length - 5); | 727 testPath.substring(dartDir.length + 1, testPath.length - 5); |
| 743 testUniqueName = testUniqueName.replaceAll('/', '_'); | 728 testUniqueName = testUniqueName.replaceAll('/', '_'); |
| 744 if (!optionsName.isEmpty()) { | 729 if (!optionsName.isEmpty()) { |
| 745 testUniqueName += '-$optionsName'; | 730 testUniqueName += '-$optionsName'; |
| 746 } | 731 } |
| 747 | 732 |
| 748 // Create '[build dir]/generated_tests/$compiler-$runtime/$testUniqueName', | 733 // Create '[build dir]/generated_tests/$compiler-$runtime/$testUniqueName', |
| 749 // including any intermediate directories that don't exist. | 734 // including any intermediate directories that don't exist. |
| 750 var generatedTestPath = ['generated_tests', | 735 var generatedTestPath = ['generated_tests', |
| 751 configuration['compiler'] + '-' + | 736 configuration['compiler'] + '-' + |
| 752 configuration['runtime'], | 737 configuration['runtime'], |
| 753 testUniqueName]; | 738 testUniqueName]; |
| 754 | 739 |
| 755 String tempDirPath = TestUtils.buildDir(configuration); | 740 String tempDirPath = TestUtils.buildDir(configuration); |
| 756 if (requiresCleanTemporaryDirectory) { | |
| 757 tempDirPath = globalTemporaryDirectory(); | |
| 758 String debugMode = | |
| 759 (configuration['mode'] == 'debug') ? 'Debug_' : 'Release_'; | |
| 760 var temp = ['${debugMode}_${configuration["arch"]}']; | |
| 761 temp.addAll(generatedTestPath); | |
| 762 generatedTestPath = temp; | |
| 763 } | |
| 764 Directory tempDir = new Directory(tempDirPath); | 741 Directory tempDir = new Directory(tempDirPath); |
| 765 if (!tempDir.existsSync()) { | 742 if (!tempDir.existsSync()) { |
| 743 // TODO(whesse): Replace this with mkdirRecursive. |
| 766 // Dartium tests can be run with no build step, with no output directory. | 744 // Dartium tests can be run with no build step, with no output directory. |
| 767 // This special case builds the build directory that should be there. | 745 // This special case builds the build directory that should be there. |
| 768 var buildPath = tempDirPath.split('/'); | 746 var buildPath = tempDirPath.split('/'); |
| 769 tempDirPath = buildPath[0]; | 747 tempDirPath = buildPath[0]; |
| 770 if (tempDirPath == '') { | 748 if (tempDirPath == '') { |
| 771 throw new Exception( | 749 throw new Exception( |
| 772 'Non-relative path to build directory in test_suite.dart'); | 750 'Non-relative path to build directory in test_suite.dart'); |
| 773 } | 751 } |
| 774 if (buildPath.length > 1) { | 752 if (buildPath.length > 1) { |
| 775 buildPath.removeRange(0, 1); | 753 buildPath.removeRange(0, 1); |
| (...skipping 311 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1087 String this.directoryPath, | 1065 String this.directoryPath, |
| 1088 String this.statusFilePath) | 1066 String this.statusFilePath) |
| 1089 : dartDir = TestUtils.dartDir(); | 1067 : dartDir = TestUtils.dartDir(); |
| 1090 | 1068 |
| 1091 bool isTestFile(String filename) => filename.endsWith("Tests.java") && | 1069 bool isTestFile(String filename) => filename.endsWith("Tests.java") && |
| 1092 !filename.contains('com/google/dart/compiler/vm') && | 1070 !filename.contains('com/google/dart/compiler/vm') && |
| 1093 !filename.contains('com/google/dart/corelib/SharedTests.java'); | 1071 !filename.contains('com/google/dart/corelib/SharedTests.java'); |
| 1094 | 1072 |
| 1095 void forEachTest(Function onTest, | 1073 void forEachTest(Function onTest, |
| 1096 Map testCacheIgnored, | 1074 Map testCacheIgnored, |
| 1097 String globalTempDir(), | |
| 1098 [Function onDone = null]) { | 1075 [Function onDone = null]) { |
| 1099 doTest = onTest; | 1076 doTest = onTest; |
| 1100 doDone = (onDone != null) ? onDone : (() => null); | 1077 doDone = (onDone != null) ? onDone : (() => null); |
| 1101 | 1078 |
| 1102 if (configuration['compiler'] != 'dartc') { | 1079 if (configuration['compiler'] != 'dartc') { |
| 1103 // Do nothing. Asynchronously report that the suite is enqueued. | 1080 // Do nothing. Asynchronously report that the suite is enqueued. |
| 1104 new Timer(0, (timerUnused){ doDone(); }); | 1081 new Timer(0, (timerUnused){ doDone(); }); |
| 1105 return; | 1082 return; |
| 1106 } | 1083 } |
| 1107 RegExp pattern = configuration['selectors']['dartc']; | 1084 RegExp pattern = configuration['selectors']['dartc']; |
| (...skipping 295 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1403 * $noCrash tests are expected to be flaky but not crash | 1380 * $noCrash tests are expected to be flaky but not crash |
| 1404 * $pass tests are expected to pass | 1381 * $pass tests are expected to pass |
| 1405 * $failOk tests are expected to fail that we won't fix | 1382 * $failOk tests are expected to fail that we won't fix |
| 1406 * $fail tests are expected to fail that we should fix | 1383 * $fail tests are expected to fail that we should fix |
| 1407 * $crash tests are expected to crash that we should fix | 1384 * $crash tests are expected to crash that we should fix |
| 1408 * $timeout tests are allowed to timeout | 1385 * $timeout tests are allowed to timeout |
| 1409 """; | 1386 """; |
| 1410 print(report); | 1387 print(report); |
| 1411 } | 1388 } |
| 1412 } | 1389 } |
| OLD | NEW |