| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
| 7 * | 7 * |
| 8 * This library includes: | 8 * This library includes: |
| 9 * | 9 * |
| 10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
| 11 * and creating [TestCase]s for those files that meet the relevant criteria. | 11 * and creating [TestCase]s for those files that meet the relevant criteria. |
| 12 * - Preparing tests, including copying files and frameworks to temporary | 12 * - Preparing tests, including copying files and frameworks to temporary |
| 13 * directories, and computing the command line and arguments to be run. | 13 * directories, and computing the command line and arguments to be run. |
| 14 */ | 14 */ |
| 15 #library("test_suite"); | 15 #library("test_suite"); |
| 16 | 16 |
| 17 #import("dart:io"); | 17 #import("dart:io"); |
| 18 #import("dart:builtin"); | 18 #import("dart:builtin"); |
| 19 #import("dart:isolate"); | 19 #import("dart:isolate"); |
| 20 #import("status_file_parser.dart"); | 20 #import("status_file_parser.dart"); |
| 21 #import("test_runner.dart"); | 21 #import("test_runner.dart"); |
| 22 #import("multitest.dart"); | 22 #import("multitest.dart"); |
| 23 #import("drt_updater.dart"); |
| 23 | 24 |
| 24 #source("browser_test.dart"); | 25 #source("browser_test.dart"); |
| 25 | 26 |
| 26 | 27 |
| 27 /** | 28 /** |
| 28 * A TestSuite represents a collection of tests. It creates a [TestCase] | 29 * A TestSuite represents a collection of tests. It creates a [TestCase] |
| 29 * object for each test to be run, and passes the test cases to a callback. | 30 * object for each test to be run, and passes the test cases to a callback. |
| 30 * | 31 * |
| 31 * Most TestSuites represent a directory or directory tree containing tests, | 32 * Most TestSuites represent a directory or directory tree containing tests, |
| 32 * and a status file containing the expected results when these tests are run. | 33 * and a status file containing the expected results when these tests are run. |
| (...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 216 bool isTestFile(String filename) => filename.endsWith("Test.dart"); | 217 bool isTestFile(String filename) => filename.endsWith("Test.dart"); |
| 217 | 218 |
| 218 bool listRecursively() => false; | 219 bool listRecursively() => false; |
| 219 | 220 |
| 220 String shellPath() => TestUtils.dartShellFileName(configuration); | 221 String shellPath() => TestUtils.dartShellFileName(configuration); |
| 221 | 222 |
| 222 List<String> additionalOptions(String filename) => []; | 223 List<String> additionalOptions(String filename) => []; |
| 223 | 224 |
| 224 void forEachTest(Function onTest, Map testCache, String globalTempDir(), | 225 void forEachTest(Function onTest, Map testCache, String globalTempDir(), |
| 225 [Function onDone = null]) { | 226 [Function onDone = null]) { |
| 227 // If DumpRenderTree is required, and not yet updated, wait for update. |
| 228 if (DumpRenderTreeUpdater.componentRequiresDRT(configuration['component']) |
| 229 && !DumpRenderTreeUpdater.updated) { |
| 230 Expect.isTrue(DumpRenderTreeUpdater.isActive); |
| 231 DumpRenderTreeUpdater.onUpdated.add(() { |
| 232 forEachTest(onTest, testCache, globalTempDir, onDone); |
| 233 }); |
| 234 return; |
| 235 } |
| 236 |
| 226 doTest = onTest; | 237 doTest = onTest; |
| 227 doDone = (onDone != null) ? onDone : (() => null); | 238 doDone = (onDone != null) ? onDone : (() => null); |
| 228 globalTemporaryDirectory = globalTempDir; | 239 globalTemporaryDirectory = globalTempDir; |
| 229 | 240 |
| 230 var filesRead = 0; | 241 var filesRead = 0; |
| 231 void statusFileRead() { | 242 void statusFileRead() { |
| 232 filesRead++; | 243 filesRead++; |
| 233 if (filesRead == statusFilePaths.length) { | 244 if (filesRead == statusFilePaths.length) { |
| 234 // Checked if we have already found and generated the tests for | 245 // Checked if we have already found and generated the tests for |
| 235 // this suite. | 246 // this suite. |
| (...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 319 } | 330 } |
| 320 Set<String> expectations = testExpectations.expectations(testName); | 331 Set<String> expectations = testExpectations.expectations(testName); |
| 321 if (configuration['report']) { | 332 if (configuration['report']) { |
| 322 // Tests with multiple VMOptions are counted more than once. | 333 // Tests with multiple VMOptions are counted more than once. |
| 323 for (var dummy in optionsFromFile["vmOptions"]) { | 334 for (var dummy in optionsFromFile["vmOptions"]) { |
| 324 SummaryReport.add(expectations); | 335 SummaryReport.add(expectations); |
| 325 } | 336 } |
| 326 } | 337 } |
| 327 if (expectations.contains(SKIP)) return; | 338 if (expectations.contains(SKIP)) return; |
| 328 | 339 |
| 329 switch (configuration['component']) { | 340 if (TestUtils.isBrowserComponent(configuration['component'])) { |
| 330 case 'dartium': | 341 enqueueBrowserTest(info, testName, expectations); |
| 331 case 'chromium': | 342 } else { |
| 332 case 'frogium': | 343 enqueueStandardTest(info, testName, expectations); |
| 333 case 'legium': | |
| 334 case 'webdriver': | |
| 335 enqueueBrowserTest(filename, testName, optionsFromFile, | |
| 336 expectations, isNegative); | |
| 337 break; | |
| 338 default: | |
| 339 isNegative = isNegative || | |
| 340 (configuration['checked'] && info.isNegativeIfChecked); | |
| 341 | |
| 342 if (configuration['component'] == 'dartc') { | |
| 343 // dartc can detect static type warnings by the | |
| 344 // format of the error line | |
| 345 if (info.hasFatalTypeErrors) { | |
| 346 isNegative = true; | |
| 347 } else if (info.hasRuntimeErrors) { | |
| 348 isNegative = false; | |
| 349 } | |
| 350 } | |
| 351 | |
| 352 var argumentLists = argumentListsFromFile(filename, | |
| 353 optionsFromFile); | |
| 354 | |
| 355 for (var args in argumentLists) { | |
| 356 doTest(new TestCase('$suiteName/$testName', | |
| 357 [new Command(shellPath(), args)], | |
| 358 configuration, | |
| 359 completeHandler, | |
| 360 expectations, | |
| 361 isNegative, | |
| 362 info)); | |
| 363 } | |
| 364 } | 344 } |
| 365 } | 345 } |
| 366 | 346 |
| 347 void enqueueStandardTest(TestInformation info, |
| 348 String testName, |
| 349 Set<String> expectations) { |
| 350 bool isNegative = info.isNegative || |
| 351 (configuration['checked'] && info.isNegativeIfChecked); |
| 352 |
| 353 if (configuration['component'] == 'dartc') { |
| 354 // dartc can detect static type warnings by the |
| 355 // format of the error line |
| 356 if (info.hasFatalTypeErrors) { |
| 357 isNegative = true; |
| 358 } else if (info.hasRuntimeErrors) { |
| 359 isNegative = false; |
| 360 } |
| 361 } |
| 362 |
| 363 var argumentLists = argumentListsFromFile(info.filename, |
| 364 info.optionsFromFile); |
| 365 |
| 366 for (var args in argumentLists) { |
| 367 doTest(new TestCase('$suiteName/$testName', |
| 368 [new Command(shellPath(), args)], |
| 369 configuration, |
| 370 completeHandler, |
| 371 expectations, |
| 372 isNegative, |
| 373 info)); |
| 374 } |
| 375 } |
| 376 |
| 367 Function makeTestCaseCreator(Map optionsFromFile) { | 377 Function makeTestCaseCreator(Map optionsFromFile) { |
| 368 return (String filename, | 378 return (String filename, |
| 369 bool isNegative, | 379 bool isNegative, |
| 370 [bool isNegativeIfChecked = false, | 380 [bool isNegativeIfChecked = false, |
| 371 bool hasFatalTypeErrors = false, | 381 bool hasFatalTypeErrors = false, |
| 372 bool hasRuntimeErrors = false, | 382 bool hasRuntimeErrors = false, |
| 373 Set<String> multitestOutcome = null]) { | 383 Set<String> multitestOutcome = null]) { |
| 374 // Cache the test information for each test case. | 384 // Cache the test information for each test case. |
| 375 var info = new TestInformation(filename, | 385 var info = new TestInformation(filename, |
| 376 optionsFromFile, | 386 optionsFromFile, |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 411 * The [StandardTestSuite] has support for testing components that | 421 * The [StandardTestSuite] has support for testing components that |
| 412 * compile a test from Dart to Javascript, and then run the resulting | 422 * compile a test from Dart to Javascript, and then run the resulting |
| 413 * Javascript. This function creates a working directory to hold the | 423 * Javascript. This function creates a working directory to hold the |
| 414 * Javascript version of the test, and copies the appropriate framework | 424 * Javascript version of the test, and copies the appropriate framework |
| 415 * files to that directory. It creates a [BrowserTestCase], which has | 425 * files to that directory. It creates a [BrowserTestCase], which has |
| 416 * two sequential steps to be run by the [ProcessQueue when] the test is | 426 * two sequential steps to be run by the [ProcessQueue when] the test is |
| 417 * executed: a compilation | 427 * executed: a compilation |
| 418 * step and an execution step, both with the appropriate executable and | 428 * step and an execution step, both with the appropriate executable and |
| 419 * arguments. | 429 * arguments. |
| 420 */ | 430 */ |
| 421 void enqueueBrowserTest(String filename, | 431 void enqueueBrowserTest(TestInformation info, |
| 422 String testName, | 432 String testName, |
| 423 Map optionsFromFile, | 433 Set<String> expectations) { |
| 424 Set<String> expectations, | 434 Map optionsFromFile = info.optionsFromFile; |
| 425 bool isNegative) { | 435 String filename = info.filename; |
| 426 if (optionsFromFile['isMultitest']) return; | 436 if (optionsFromFile['isMultitest']) return; |
| 427 bool isWebTest = optionsFromFile['containsDomImport']; | 437 bool isWebTest = optionsFromFile['containsDomImport']; |
| 428 bool isLibraryDefinition = optionsFromFile['isLibraryDefinition']; | 438 bool isLibraryDefinition = optionsFromFile['isLibraryDefinition']; |
| 429 if (!isLibraryDefinition && optionsFromFile['containsSourceOrImport']) { | 439 if (!isLibraryDefinition && optionsFromFile['containsSourceOrImport']) { |
| 430 print('Warning for $filename: Browser tests require #library ' + | 440 print('Warning for $filename: Browser tests require #library ' + |
| 431 'in any file that uses #import, #source, or #resource'); | 441 'in any file that uses #import, #source, or #resource'); |
| 432 } | 442 } |
| 433 | 443 |
| 434 final String component = configuration['component']; | 444 final String component = configuration['component']; |
| 445 Expect.isTrue(DumpRenderTreeUpdater.componentRequiresDRT(component)); |
| 435 final String testPath = | 446 final String testPath = |
| 436 new File(filename).fullPathSync().replaceAll('\\', '/'); | 447 new File(filename).fullPathSync().replaceAll('\\', '/'); |
| 437 | 448 |
| 438 for (var vmOptions in optionsFromFile['vmOptions']) { | 449 for (var vmOptions in optionsFromFile['vmOptions']) { |
| 439 // Create a unique temporary directory for each set of vmOptions. | 450 // Create a unique temporary directory for each set of vmOptions. |
| 440 // TODO(dart:429): Replace separate replaceAlls with a RegExp when | 451 // TODO(dart:429): Replace separate replaceAlls with a RegExp when |
| 441 // replaceAll(RegExp, String) is implemented. | 452 // replaceAll(RegExp, String) is implemented. |
| 442 String optionsName = ''; | 453 String optionsName = ''; |
| 443 if (optionsFromFile['vmOptions'].length > 1) { | 454 if (optionsFromFile['vmOptions'].length > 1) { |
| 444 optionsName = Strings.join(vmOptions, '-').replaceAll('-','') | 455 optionsName = Strings.join(vmOptions, '-').replaceAll('-','') |
| (...skipping 191 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 636 // including any intermediate directories that don't exist. | 647 // including any intermediate directories that don't exist. |
| 637 var generatedTestPath = ['generated_tests', | 648 var generatedTestPath = ['generated_tests', |
| 638 configuration['component'], | 649 configuration['component'], |
| 639 testUniqueName]; | 650 testUniqueName]; |
| 640 | 651 |
| 641 String tempDirPath = TestUtils.buildDir(configuration); | 652 String tempDirPath = TestUtils.buildDir(configuration); |
| 642 if (requiresCleanTemporaryDirectory) { | 653 if (requiresCleanTemporaryDirectory) { |
| 643 tempDirPath = globalTemporaryDirectory(); | 654 tempDirPath = globalTemporaryDirectory(); |
| 644 String debugMode = | 655 String debugMode = |
| 645 (configuration['mode'] == 'debug') ? 'Debug_' : 'Release_'; | 656 (configuration['mode'] == 'debug') ? 'Debug_' : 'Release_'; |
| 646 generatedTestPath = ['${debugMode}_${configuration["arch"]}'] | 657 var temp = ['${debugMode}_${configuration["arch"]}']; |
| 647 .addAll(generatedTestPath); | 658 temp.addAll(generatedTestPath); |
| 659 generatedTestPath = temp; |
| 648 } | 660 } |
| 649 Directory tempDir = new Directory(tempDirPath); | 661 Directory tempDir = new Directory(tempDirPath); |
| 650 if (!tempDir.existsSync()) { | 662 if (!tempDir.existsSync()) { |
| 651 // Dartium tests can be run with no build step, with no output directory. | 663 // Dartium tests can be run with no build step, with no output directory. |
| 652 // This special case builds the build directory that should be there. | 664 // This special case builds the build directory that should be there. |
| 653 var buildPath = tempDirPath.split('/'); | 665 var buildPath = tempDirPath.split('/'); |
| 654 tempDirPath = buildPath[0]; | 666 tempDirPath = buildPath[0]; |
| 655 if (tempDirPath == '') { | 667 if (tempDirPath == '') { |
| 656 throw new Exception( | 668 throw new Exception( |
| 657 'Non-relative path to build directory in test_suite.dart'); | 669 'Non-relative path to build directory in test_suite.dart'); |
| (...skipping 506 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1164 args.add('--enable_asserts'); | 1176 args.add('--enable_asserts'); |
| 1165 args.add("--enable_type_checks"); | 1177 args.add("--enable_type_checks"); |
| 1166 } | 1178 } |
| 1167 if (configuration["component"] == "leg" | 1179 if (configuration["component"] == "leg" |
| 1168 || configuration["component"] == "legium") { | 1180 || configuration["component"] == "legium") { |
| 1169 args.add("--verbose"); | 1181 args.add("--verbose"); |
| 1170 args.add("--leg"); | 1182 args.add("--leg"); |
| 1171 } | 1183 } |
| 1172 return args; | 1184 return args; |
| 1173 } | 1185 } |
| 1186 |
| 1187 static bool isBrowserComponent(String component) => |
| 1188 const <String>['dartium', |
| 1189 'frogium', |
| 1190 'legium', |
| 1191 'webdriver'].some((x) => x == component); |
| 1174 } | 1192 } |
| 1175 | 1193 |
| 1176 class SummaryReport { | 1194 class SummaryReport { |
| 1177 static int total = 0; | 1195 static int total = 0; |
| 1178 static int skipped = 0; | 1196 static int skipped = 0; |
| 1179 static int noCrash = 0; | 1197 static int noCrash = 0; |
| 1180 static int pass = 0; | 1198 static int pass = 0; |
| 1181 static int failOk = 0; | 1199 static int failOk = 0; |
| 1182 static int fail = 0; | 1200 static int fail = 0; |
| 1183 static int crash = 0; | 1201 static int crash = 0; |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1217 * $noCrash tests are expected to be flaky but not crash | 1235 * $noCrash tests are expected to be flaky but not crash |
| 1218 * $pass tests are expected to pass | 1236 * $pass tests are expected to pass |
| 1219 * $failOk tests are expected to fail that we won't fix | 1237 * $failOk tests are expected to fail that we won't fix |
| 1220 * $fail tests are expected to fail that we should fix | 1238 * $fail tests are expected to fail that we should fix |
| 1221 * $crash tests are expected to crash that we should fix | 1239 * $crash tests are expected to crash that we should fix |
| 1222 * $timeout tests are allowed to timeout | 1240 * $timeout tests are allowed to timeout |
| 1223 """; | 1241 """; |
| 1224 print(report); | 1242 print(report); |
| 1225 } | 1243 } |
| 1226 } | 1244 } |
| OLD | NEW |