Add support for Dromaeo tests in perf scripts.

tools/testing/run_selenium.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 #
 
 """Script to actually open a browser and perform the test, and reports back with
 the result. It uses Selenium WebDriver when possible for running the tests. It
 uses Selenium RC for Safari.
@@ skipping 15 matching lines @@
 stdin:  --terminate
 $
 """
 
 import os
 import optparse
 import platform
 import selenium
 from selenium.webdriver.support.ui import WebDriverWait
 import shutil
+import signal
 import socket
 import sys
 import time
-import signal
 
 TIMEOUT_ERROR_MSG = 'FAIL (timeout)'
 
-def perf_test_done(driver):
-  """Checks if the performance test has completed."""
-  return perf_test_done_helper(driver.page_source)
+def correctness_test_done(source):
+  """Checks if test has completed."""
+  return ('PASS' in source) or ('FAIL' in source)
 
-def perf_test_done_helper(source):
+def perf_test_done(source):
   """Tests to see if our performance test is done by printing a score."""
   #This code is written this way to work around a current instability in the
   # python webdriver bindings if you call driver.get_element_by_id.
   #TODO(efortuna): Access these elements in a nicer way using DOM parser.
   string = '<div id="status">'
   index = source.find(string)
   end_index = source.find('</div>', index+1)
   source = source[index + len(string):end_index]
   return 'Score:' in source
 
-def run_test_in_browser(browser, html_out, timeout, is_perf):
+def dromaeo_test_done(source):
+  """Tests to see if our performance test is done by printing a score."""
+  #TODO(efortuna): Access these elements in a nicer way using DOM parser.
+  string = '<span class="left">'
+  index = source.find(string)
+  end_index = source.find('</span>', index+1)
+  source = source[index + len(string):end_index]
+  return '0:00' in source
+
+# TODO(vsm): Ideally, this wouldn't live in this file.
+CONFIGURATIONS = {
+    'correctness': correctness_test_done,
+    'perf': perf_test_done,
+    'dromaeo': dromaeo_test_done
+}
+
+def run_test_in_browser(browser, html_out, timeout, config):
   """Run the desired test in the browser using Selenium 2.0 WebDriver syntax,
   and wait for the test to complete. This is the newer syntax, that currently
   supports Firefox, Chrome, IE, Opera (and some mobile browsers)."""
   if isinstance(browser, selenium.selenium):
-    return run_test_in_browser_selenium_rc(browser, html_out, timeout, is_perf)
+    return run_test_in_browser_selenium_rc(browser, html_out, timeout, config)
 
   browser.get("file://" + html_out)
   source = ''
   try:
-    if is_perf:
-      # We're running a performance test.
-      element = WebDriverWait(browser, float(timeout)).until(perf_test_done)
-    else:
-      element = WebDriverWait(browser, float(timeout)).until(
-          lambda driver : ('PASS' in driver.page_source) or
-          ('FAIL' in driver.page_source))
+    test_done = CONFIGURATIONS[config]
+    element = WebDriverWait(browser, float(timeout)).until(
+        lambda driver: test_done(driver.page_source))
     source = browser.page_source
   except selenium.common.exceptions.TimeoutException:
     source = TIMEOUT_ERROR_MSG
   return source
 
-def run_test_in_browser_selenium_rc(sel, html_out, timeout, is_perf):
+def run_test_in_browser_selenium_rc(sel, html_out, timeout, config):
   """ Run the desired test in the browser using Selenium 1.0 syntax, and wait
   for the test to complete. This is used for Safari, since it is not currently
   supported on Selenium 2.0."""
   sel.open('file://' + html_out)
   source = sel.get_html_source()
-  def end_condition(source):
-    return 'PASS' in source or 'FAIL' in source
-  if is_perf:
-    end_condition = perf_test_done_helper
+  end_condition = CONFIGURATIONS[config]
 
   elapsed = 0
   while (not end_condition(source)) and elapsed <= timeout:
     sec = .25
     time.sleep(sec)
     elapsed += sec
     source = sel.get_html_source()
   return source
 
 def parse_args(args=None):
   parser = optparse.OptionParser()
   parser.add_option('--out', dest='out',
       help = 'The path for html output file that we will running our test from',
       action = 'store', default = '')
   parser.add_option('--browser', dest='browser',
       help = 'The browser type (default = chrome)',
       action = 'store', default = 'chrome')
   # TODO(efortuna): Put this back up to be more than the default timeout in
   # test.dart. Right now it needs to be less than 60 so that when test.dart
   # times out, this script also closes the browser windows.
   parser.add_option('--timeout', dest = 'timeout',
       help = 'Amount of time (seconds) to wait before timeout', type = 'int',
       action = 'store', default=58)
   parser.add_option('--perf', dest = 'is_perf',
       help = 'Add this flag if we are running a browser performance test',
       action = 'store_true', default=False)
+  # TODO(vsm): Abstract this out better.
+  parser.add_option('--dromaeo', dest = 'is_dromaeo',
+      help = 'Add this flag if we are running a browser performance test',
+      action = 'store_true', default=False)
   args, ignored = parser.parse_args(args=args)
-  return args.out, args.browser, args.timeout, args.is_perf
+  if args.is_perf:
+    config = 'perf'
+  elif args.is_dromaeo:
+    config = 'dromaeo'
+  else:
+    config = 'correctness'
+  return args.out, args.browser, args.timeout, config
 
 def start_browser(browser, html_out):
   if browser == 'chrome':
     # Note: you need ChromeDriver *in your path* to run Chrome, in addition to
     # installing Chrome. Also note that the build bot runs have a different path
     # from a normal user -- check the build logs.
     return selenium.webdriver.Chrome()
   elif browser == 'ff':
     profile = selenium.webdriver.firefox.firefox_profile.FirefoxProfile()
     profile.set_preference('dom.max_script_run_time', 0)
@@ skipping 30 matching lines @@
 
   # A timeout exception is thrown if nothing happens within the time limit.
   if browser != 'chrome':
     browser.close()
   try:
     browser.quit()
   except selenium.common.exceptions.WebDriverException:
     # TODO(efortuna): Figure out why this crashes.... and avoid?
     pass
 
-def report_results(is_perf, source):
-  if is_perf:
+def report_results(config, source):
+  # TODO(vsm): Add a failure check for Dromaeo.
+  if config != 'correctness':
     # We're running a performance test.
-    print source
+    print source.encode('utf8')
+    sys.stdout.flush()
     if 'NaN' in source:
       return 1
     else:
       return 0
   else:
     # We're running a correctness test. Mark test as passing if all individual
     # test cases pass.
     if 'FAIL' not in source and 'PASS' in source:
       print 'Content-Type: text/plain\nPASS'
       return 0
@@ skipping 35 matching lines @@
   # TODO(jmesserly): It'd be nice to shutdown gracefully in the event of a
   # SIGTERM. Unfortunately dart:io cannot send SIGTERM, see dartbug.com/1756.
   signal.signal(signal.SIGTERM, lambda number, frame: close_browser(browser))
 
   try:
     while True:
       line = sys.stdin.readline()
       if line == '--terminate\n':
         break
 
-      html_out, browser_name, timeout, is_perf = parse_args(line.split())
+      html_out, browser_name, timeout, config = parse_args(line.split())
 
       # Sanity checks that test.dart is passing flags we can handle.
-      if is_perf:
+      if config != 'correctness':
         print 'Batch test runner not compatible with perf testing'
         return 1
       if browser and current_browser_name != browser_name:
         print('Batch test runner got multiple browsers: %s and %s'
             % (current_browser_name, browser_name))
         return 1
 
       # Start the browser on the first run
       if browser is None:
         current_browser_name = browser_name
         browser = start_browser(browser_name, html_out)
 
-      source = run_test_in_browser(browser, html_out, timeout, is_perf)
+      source = run_test_in_browser(browser, html_out, timeout, config)
 
       # Test is done. Write end token to stderr and flush.
       sys.stderr.write('>>> EOF STDERR\n')
       sys.stderr.flush()
 
       # print one of:
       # >>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT}
-      status = report_results(is_perf, source)
+      status = report_results(config, source)
       if status == 0:
         print '>>> TEST PASS'
       elif source == TIMEOUT_ERROR_MSG:
         print '>>> TEST TIMEOUT'
       else:
         print '>>> TEST FAIL'
       sys.stdout.flush()
   finally:
     close_browser(browser)
 
 
 def main(args):
   # Run in batch mode if the --batch flag is passed.
   # TODO(jmesserly): reconcile with the existing args parsing
   if '--batch' in args:
     return run_batch_tests()
 
   # Run a single test
-  html_out, browser_name, timeout, is_perf = parse_args()
+  html_out, browser_name, timeout, config = parse_args()
   browser = start_browser(browser_name, html_out)
 
   try:
-    output = run_test_in_browser(browser, html_out, timeout, is_perf)
-    return report_results(is_perf, output)
+    output = run_test_in_browser(browser, html_out, timeout, config)
+    return report_results(config, output)
   finally:
     close_browser(browser)
 
 if __name__ == "__main__":
   sys.exit(main(sys.argv))