Rename test.dart component to specify compiler + runtime.

tools/testing/dart/test_options.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #library("test_options_parser");
 
 #import("dart:io");
 #import("dart:builtin");
 #import("drt_updater.dart");
 
@@ skipping 44 matching lines @@
    */
   TestOptionsParser() {
     _options =
         [ new _TestOptionSpecification(
               'mode',
               'Mode in which to run the tests',
               ['-m', '--mode'],
               ['all', 'debug', 'release'],
               'debug'),
           new _TestOptionSpecification(
-              'component',
-              '''
-Controls how dart code is compiled and executed.
+              'compiler',
+              '''Specify any compilation step (if needed).
 
-   vm: Run dart code on the standalone dart vm.
+   none: Do not compile the Dart code (run native Dart code on the VM).
 
-   frog: Compile dart code by running frog on the standalone dart vm, and
-       run the resulting javascript on D8.
+   frog: Compile dart code to JavaScript by running the frog compiler.
 
-   dart2js: Compile dart code by running dart2js on the standalone
-       dart vm, and run the resulting javascript on D8.
+   dart2js: Compile dart code to JavaScript by running dart2js (leg).
 
-   frogsh: Compile dart code by running frog on node.js, and run the
-       resulting javascript on the same instance of node.js.
-
-   dartium: Run dart code in a type="application/dart" script tag in a
-       dartium build of DumpRenderTree.
-
-   frogium: Compile dart code by running frog on the standalone dart vm,
-       and run the resulting javascript in a javascript script tag in
-       a dartium build of DumpRenderTree.
-
-   legium: Compile dart code by running dart2js on the standalone dart
-       vm, and run the resulting javascript in a javascript script tag
-       in a dartium build of DumpRenderTree.
-
-   webdriver: Compile dart code by running frog on the standalone dart vm,
-       and then run the resulting javascript in the browser that is specified
-       by the --browser switch (e.g. chrome, safari, ff, etc.).
-
-   dartc: Run dart code through the dartc static analyzer (does not
-       execute dart code).
-''',
-              ['-c', '--component'],
-              ['most', 'vm', 'frog', 'dart2js', 'frogsh', 'dartium',  'frogium',
-               'legium', 'webdriver', 'dartc'],
+   dartc: Perform static analysis on Dart code by running dartc.
+   
+   frogsh: Compile dart code to JavaScript by running the frog compiler on
+           node.js, and run the resulting JavaScript on the same instance of 
+           node.js.''',

[Siggi Cherem (dart-lang), 2012/03/26 22:28:40]

since compiler and runtime have limited combinations together, maybe we can add
to this comment (and below to the runtime comment) which are valid combinations?
For instance:

none: Do not .... 
  only valid with with the following runtimes: vm, drt

frog: Compile ...
  valid with the following runtimes: d8, drt, chrome, ...

dartc: Perform ...
  only valid with the runtime: none

+              ['-c', '--compiler'],
+              ['none', 'frog', 'dart2js', 'dartc', 'frogsh'],
+              'none'),
+          new _TestOptionSpecification(
+              'runtime',
+              '''Where the tests should be run.
+    vm: Run Dart code on the standalone dart vm.
+    
+    d8: Run JavaScript from the command line using v8.
+    
+    drt: Run Dart or JavaScript in the headless version of Chrome, 
+         DumpRenderTree.
+    
+    ff or firefox: Run JavaScript in Firefox.
+    
+    chrome: Run JavaScript in Chrome.
+    
+    safari: Run JavaScript in Safari.
+    
+    ie: Run JavaScript in Internet Explorer.
+    
+    opera: Run JavaScript in Opera.
+    
+    none: No runtime, compile only (for example, used for dartc static analysis 
+          tests).''',
+              ['-r', '--runtime'],
+              ['vm', 'd8', 'drt', 'ff', 'firefox', 'chrome', 
+              'safari', 'ie', 'opera', 'none'],
               'vm'),
           new _TestOptionSpecification(
               'arch',
               'The architecture to run tests for',
               ['-a', '--arch'],
               ['all', 'ia32', 'x64', 'simarm'],
               'ia32'),
           new _TestOptionSpecification(
               'system',
               'The operating system to run tests on',
@@ skipping 84 matching lines @@
               [],
               ''),
           new _TestOptionSpecification(
               'time',
               'Print timing information after running tests',
               ['--time'],
               [],
               false,
               'bool'),
           new _TestOptionSpecification(
-              'browser',
-              'Web browser to use on webdriver tests',
-              ['-b', '--browser'],
-              ['ff', 'chrome', 'safari', 'ie', 'opera'],
-              'chrome'),
-          new _TestOptionSpecification(
-              'frog',
-              'Path to frog executable',
-              ['--frog'],
-              [],
-              ''),
-          new _TestOptionSpecification(
               'drt',
               'Path to DumpRenderTree executable',
               ['--drt'],
               [],
               ''),
           new _TestOptionSpecification(
-              'froglib',
-              'Path to frog library',
-              ['--froglib'],
-              [],
-              ''),
-          new _TestOptionSpecification(
               'noBatch',
               'Do not run browser tests in batch mode',
               ['-n', '--nobatch'],
               [],
               false,
               'bool')];
   }
 
 
   /**
@@ skipping 103 matching lines @@
       }
     }
 
     // Apply default values for unspecified options.
     for (var option in _options) {
       if (!configuration.containsKey(option.name)) {
         configuration[option.name] = option.defaultValue;
       }
     }
 
-    return _expandConfigurations(configuration);
+    List<Map> expandedConfigs = _expandConfigurations(configuration);
+    return expandedConfigs.filter(_isValidConfig);
   }
 
+  /**
+   * Loop through the set of configurations that we will be testing. Return a

[Siggi Cherem (dart-lang), 2012/03/26 22:28:40]

rewrite comment to talk about a single config? (the current comment seems
appropriate for 'filter' above where you are iterating through the configs).

[Emily Fortuna, 2012/03/26 23:31:32]

Done.

+   * list that contains the valid set of configurations that will actually be
+   * tested.
+   */
+  bool _isValidConfig(Map config) {
+    bool isValid = true;
+      switch (config['compiler']) {
+        case 'frog':
+        case 'dart2js':
+        case 'frogsh':
+          isValid = (const ['d8', 'drt', 'ff', 'chrome', 'safari', 'ie',

[Siggi Cherem (dart-lang), 2012/03/26 22:48:04]

btw - is the current os/system available in [config] as well? (e.g. can we later
check that ie is only valid if you are in windows?)

+              'opera', 'none']).indexOf(config['runtime']) >= 0;

[Siggi Cherem (dart-lang), 2012/03/26 22:28:40]

I thought [none] wasn't valid here. I guess if you just want to see that it
compiles things correctly this would be ok. Is that the intention here?

[Siggi Cherem (dart-lang), 2012/03/26 22:48:04]

BTW - One disadvantage of adding 'none' here is that we wont be able to run some
combinations together that easily. For instance, before we were able to do:

test.py -c frog,dart2js,dartc -r drt,none

to run these configurations:
frog_drt
dart2js_drt
dartc_none

With this change we will repeat work tested above by running also:
frog_none
dart2js_none

I think this is ok, but if later we don't need 'none' with dart2js anymore, I'd
remove it from here.

[Emily Fortuna, 2012/03/26 23:31:32]

I added this at Peter's request. Peter wanted to add "none" for ensuring that
their tests compile correctly with dart:io (I think?)

[Emily Fortuna, 2012/03/26 23:31:32]

Added a comment.

+          break;
+        case 'dartc':
+          isValid = config['runtime'] == 'none';
+          break;
+        case 'none':
+          isValid = (const ['vm', 'drt']).indexOf(config['runtime']) >= 0;
+      }
+    if (!isValid) {
+      print("Warning: combination of ${config['compiler']} and " +
+          "${config['runtime']} is invalid. Skipping this combination.");
+    }
+    return isValid;
+  }
 
   /**
    * Recursively expand a configuration with multiple values per key
    * into a list of configurations with exactly one value per key.
    */
   List<Map> _expandConfigurations(Map configuration) {
     // Expand the pseudo-values such as 'all'.
     if (configuration['arch'] == 'all') {
       configuration['arch'] = 'ia32,x64';
     }
     if (configuration['mode'] == 'all') {
       configuration['mode'] = 'debug,release';
     }
-    if (configuration['component'] == 'most') {
-      configuration['component'] = 'vm,dartc';
-    }
     if (configuration['valgrind']) {
       // TODO(ager): Get rid of this when there is only one checkout and
       // we don't have to special case for the runtime checkout.
       File valgrindFile = new File('runtime/tools/valgrind.py');
       if (!valgrindFile.existsSync()) {
         valgrindFile = new File('../runtime/tools/valgrind.py');
       }
       String valgrind = valgrindFile.fullPathSync();
       configuration['special-command'] = 'python -u $valgrind @';
     }
 
     // Use verbose progress indication for verbose output unless buildbot
     // progress indication is requested.
     if (configuration['verbose'] && configuration['progress'] != 'buildbot') {
       configuration['progress'] = 'verbose';
     }
 
     // Create the artificial 'unchecked' options that test status files
     // expect.
     configuration['unchecked'] = !configuration['checked'];
     configuration['host_unchecked'] = !configuration['host_checked'];
 
+    if (configuration['runtime'] == 'firefox') {
+      configuration['runtime'] == 'ff';
+    }
+
     // Expand the test selectors into a suite name and a simple
     // regular expressions to be used on the full path of a test file
     // in that test suite. If no selectors are explicitly given use
     // the default suite patterns.
     var selectors = configuration['selectors'];
     if (selectors is !Map) {
       if (selectors == null) {
         selectors = new List.from(defaultTestSelectors);
       }
       Map<String, RegExp> selectorMap = new Map<String, RegExp>();
@@ skipping 13 matching lines @@
           print("Error: '$suite/$pattern'.  Only one test selection" +
                 " pattern is allowed to start with '$suite/'");
           exit(1);
         }
         selectorMap[suite] = new RegExp(pattern);
       }
       configuration['selectors'] = selectorMap;
     }
 
     // Expand the architectures.
-    var archs = configuration['arch'];
-    if (archs.contains(',')) {
-      var result = new List<Map>();
-      for (var arch in archs.split(',')) {
-        var newConfiguration = new Map.from(configuration);
-        newConfiguration['arch'] = arch;
-        result.addAll(_expandConfigurations(newConfiguration));
-      }
-      return result;
+    if (configuration['arch'].contains(',')) {
+      return _expandHelper('arch', configuration);
     }
 
     // Expand modes.
-    var modes = configuration['mode'];
-    if (modes.contains(',')) {
-      var result = new List<Map>();
-      for (var mode in modes.split(',')) {
-        var newConfiguration = new Map.from(configuration);
-        newConfiguration['mode'] = mode;
-        result.addAll(_expandConfigurations(newConfiguration));
-      }
-      return result;
+    if (configuration['mode'].contains(',')) {
+      return _expandHelper('mode', configuration);
+    }
+        
+    // Expand compilers.
+    if (configuration['compiler'].contains(',')) {
+      return _expandHelper('compiler', configuration);
     }
 
-    // Expand components.
-    var components = configuration['component'];
-    if (components.contains(',')) {
-      var result = new List<Map>();
-      for (var component in components.split(',')) {
-        var newConfiguration = new Map.from(configuration);
-        newConfiguration['component'] = component;
-        result.addAll(_expandConfigurations(newConfiguration));
-      }
-      return result;
+    // Expand runtimes.
+    var runtimes = configuration['runtime'];
+    if (runtimes.contains(',')) {
+      return _expandHelper('runtime', configuration);
     } else {
-      // All components eventually go through this path, after expansion.
-      if (DumpRenderTreeUpdater.componentRequiresDRT(components)) {
+      // All runtimes eventually go through this path, after expansion.
+      if (runtimes == 'drt') {
         DumpRenderTreeUpdater.update();
       }
     }
 
-    // Adjust default timeout based on mode and component.
+    // Adjust default timeout based on mode, compiler, and sometimes runtime.
     if (configuration['timeout'] == -1) {
       var timeout = 60;
-      switch (configuration['component']) {
+      switch (configuration['compiler']) {
         case 'dartc':
-        case 'dartium':
-        case 'frogium':
-        case 'legium':
-        case 'webdriver':
           timeout *= 4;
           break;
         case 'dart2js':
         case 'frog':
           if (configuration['mode'] == 'debug') {
             timeout *= 8;
           }
           if (configuration['host_checked']) {
             timeout *= 16;
           }
+          if ((const ['ie', 'ff', 'chrome', 'safari',
+            'opera']).indexOf(configuration['runtime']) >= 0) {
+            timeout *= 4; // Allow additional time for browser testing to run.
+          }
           break;
         default:
           if (configuration['mode'] == 'debug') {
             timeout *= 2;
           }
+          if (configuration['runtime'] == 'drt') {
+            timeout *= 4;
+          }
           break;
       }
       configuration['timeout'] = timeout;
     }
 
     return [configuration];
   }
 
+  /** 
+   * Helper for _expandConfigurations. Creates a new configuration and adds it
+   * to a list, for use in a case when a particular configuration has multiple
+   * results (separated by a ',').
+   * Arguments:
+   * option: The particular test option we are expanding.
+   * configuration: The map containing all test configuration information
+   * specified.
+   */
+  List<Map> _expandHelper(String option, Map configuration) {
+    var result = new List<Map>();
+    var configs = configuration[option]; 
+    for (var config in configs.split(',')) {
+      var newConfiguration = new Map.from(configuration);
+      newConfiguration[option] = config;
+      result.addAll(_expandConfigurations(newConfiguration));
+    }
+    return result;
+  }
+
 
   /**
    * Print out usage information.
    */
   void _printHelp() {
     print('usage: dart test.dart [options]\n');
     print('Options:\n');
     for (var option in _options) {
       print('${option.name}: ${option.description}.');
       for (var name in option.keys) {
@@ skipping 35 matching lines @@
         return option;
       }
     }
     print('Unknown test option $name');
     exit(1);
   }
 
 
   List<_TestOptionSpecification> _options;
 }