Convert runtest to have a built-in annotator.

scripts/slave/runtest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A tool to run a chrome test executable, used by the buildbot slaves.
 
   When this is run, the current directory (cwd) should be the outer build
   directory (e.g., chrome-release/build/).
 
   For a list of command-line options, call this script with '--help'.
 """
 
 import copy
 import logging
 import optparse
 import os
+import re
 import stat
 import sys
 import tempfile
 
 # sys.path needs to be modified here because python2.6 automatically adds the
 # system "google" module (/usr/lib/pymodules/python2.6/google) to sys.modules
 # when we import "chromium_config" (I don't know why it does this). This causes
 # the import of our local "google.*" modules to fail because python seems to
 # only look for a system "google.*", even if our path is in sys.path before
 # importing "google.*". If we modify sys.path here, before importing
 # "chromium_config", python2.6 properly uses our path to find our "google.*"
 # (even though it still automatically adds the system "google" module to
 # sys.modules, and probably should still be using that to resolve "google.*",
 # which I really don't understand).
 sys.path.insert(0, os.path.abspath('src/tools/python'))
 
 # Because of this dependency on a chromium checkout, we need to disable some
 # pylint checks.
 # pylint: disable=E0611
 # pylint: disable=E1101
+from buildbot.status import builder
+
 from common import chromium_utils
+from common import gtest_utils
 from slave import gtest_slave_utils
 from slave import slave_utils
 from slave import xvfb
 import config
 
 USAGE = '%s [options] test.exe [test args]' % os.path.basename(sys.argv[0])
 
 CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
 
 DEST_DIR = 'gtest_results'
 
 HTTPD_CONF = {
     'linux': 'httpd2_linux.conf',
     'mac': 'httpd2_mac.conf',
     'win': 'httpd.conf'
 }
 
+
 def should_enable_sandbox(sandbox_path):
   """Return a boolean indicating that the current slave is capable of using the
   sandbox and should enable it.  This should return True iff the slave is a
   Linux host with the sandbox file present and configured correctly."""
   if not (sys.platform.startswith('linux') and
           os.path.exists(sandbox_path)):
     return False
   sandbox_stat = os.stat(sandbox_path)
   if ((sandbox_stat.st_mode & stat.S_ISUID) and
       (sandbox_stat.st_mode & stat.S_IRUSR) and
       (sandbox_stat.st_mode & stat.S_IXUSR) and
       (sandbox_stat.st_uid == 0)):
     return True
   return False
 
+
 def get_temp_count():
   """Returns the number of files and directories inside the temporary dir."""
   return len(os.listdir(tempfile.gettempdir()))
 
 
 def _RunGTestCommand(command, results_tracker=None, pipes=None):
   if results_tracker and pipes:
     # This is not supported by RunCommand.
     print 'Invalid test invocation. (results_tracker and pipes)'
     return 1
   if results_tracker:
     return chromium_utils.RunCommand(
-        command, parser_func=results_tracker.OnReceiveLine)
+        command, parser_func=results_tracker.ProcessLine)
   else:
     return chromium_utils.RunCommand(command, pipes=pipes)
 
 
 def _GenerateJSONForTestResults(options, results_tracker):
   """Generate (update) a JSON file from the gtest results XML and
   upload the file to the archive server.
   The archived JSON file will be placed at:
   www-dir/DEST_DIR/buildname/testname/results.json
   on the archive server (NOTE: this is to be deprecated).
   Note that it adds slave's WebKit/Tools/Scripts to the PYTHONPATH
   to run the JSON generator.
 
   Args:
     options: command-line options that are supposed to have build_dir,
         results_directory, builder_name, build_name and test_output_xml values.
   """
   # pylint: disable=W0703
   results_map = None
   try:
     if os.path.exists(options.test_output_xml):
       results_map = gtest_slave_utils.GetResultsMapFromXML(
           options.test_output_xml)
     else:
       sys.stderr.write('Unable to generate JSON from XML, using log output.')
       # The file did not get generated. See if we can generate a results map
       # from the log output.
-      results_map = results_tracker.GetResultsMap()
+      results_map = gtest_slave_utils.GetResultsMap(results_tracker)
   except Exception, e:
     # This error will be caught by the following 'not results_map' statement.
     print 'Error: ', e
 
   if not results_map:
     print 'No data was available to update the JSON results'
     return
 
   build_dir = os.path.abspath(options.build_dir)
   slave_name = slave_utils.SlaveBuildName(build_dir)
@@ skipping 18 matching lines @@
 
     # Generate results JSON file and upload it to the appspot server.
     gtest_slave_utils.GenerateAndUploadJSONResults(
         results_map, generate_json_options)
 
     # The code can throw all sorts of exceptions, including
     # slave.gtest.networktransaction.NetworkTimeout so just trap everything.
   except:  # pylint: disable=W0702
     print 'Unexpected error while generating JSON'
 
+
 def _BuildParallelCommand(build_dir, test_exe_path, options):
   supervisor_path = os.path.join(build_dir, '..', 'tools',
                                  'sharding_supervisor',
                                  'sharding_supervisor.py')
   supervisor_args = ['--no-color']
   if options.factory_properties.get('retry_failed', True):
     supervisor_args.append('--retry-failed')
   if options.total_shards and options.shard_index:
     supervisor_args.extend(['--total-slaves', str(options.total_shards),
                             '--slave-index', str(options.shard_index - 1)])
   if options.sharding_args:
     supervisor_args.extend(options.sharding_args.split())
   command = [sys.executable, supervisor_path]
   command.extend(supervisor_args)
   command.append(test_exe_path)
   return command
 
+
 def start_http_server(platform, build_dir, test_exe_path, document_root):
   # pylint: disable=F0401
   import google.httpd_utils
   import google.platform_utils
   platform_util = google.platform_utils.PlatformUtility(build_dir)
 
   # Name the output directory for the exe, without its path or suffix.
   # e.g., chrome-release/httpd_logs/unit_tests/
   test_exe_name = os.path.splitext(os.path.basename(test_exe_path))[0]
   output_dir = os.path.join(slave_utils.SlaveBaseDir(build_dir),
@@ skipping 22 matching lines @@
                                                  document_root)
   stop_cmd = platform_util.GetStopHttpdCommand()
   http_server = google.httpd_utils.ApacheHttpd(start_cmd, stop_cmd, [8000])
   try:
     http_server.StartServer()
   except google.httpd_utils.HttpdNotStarted, e:
     raise google.httpd_utils.HttpdNotStarted('%s. See log file in %s' %
                                              (e, output_dir))
   return http_server
 
+
+def getText(result, observer, name):
+  """Generate a text summary for the waterfall.
+
+  Updates the waterfall with any unusual test output, with a link to logs of
+  failed test steps.
+  """
+  GTEST_DASHBOARD_BASE = ('http://test-results.appspot.com'
+    '/dashboards/flakiness_dashboard.html')
+
+  # basic_info is an array of lines to display on the waterfall
+  basic_info = [name]
+
+  disabled = observer.DisabledTests()
+  if disabled:
+    basic_info.append('%s disabled' % str(disabled))
+
+  flaky = observer.FlakyTests()
+  if flaky:
+    basic_info.append('%s flaky' % str(flaky))
+
+  failed_test_count = len(observer.FailedTests())
+
+  if failed_test_count == 0:
+    if result is builder.SUCCESS:
+      return basic_info
+    elif result is builder.WARNINGS:
+      return basic_info + ['warnings']
+
+  if observer.RunningTests():
+
+    basic_info += ['did not complete']
+
+  # TODO(xusydoc):  see if 'crashed or hung' should be tracked by RunningTests()
+  if failed_test_count:
+    failure_text = ['failed %d' % failed_test_count]
+    if observer.master_name:
+      # Include the link to the flakiness dashboard
+      failure_text.append('<div class="BuildResultInfo">')
+      failure_text.append('<a href="%s#master=%s&testType=%s'
+                          '&tests=%s">' % (
+          GTEST_DASHBOARD_BASE, observer.master_name,
+          name,
+          ','.join(observer.FailedTests())))
+      failure_text.append('Flakiness dashboard')
+      failure_text.append('</a>')
+      failure_text.append('</div>')
+  else:
+    failure_text = ['crashed or hung']
+  return basic_info + failure_text
+
+
+def annotate(test_name, result, results_tracker):
+  """Given a test result and tracker, update the waterfall with test results."""
+  get_text_result = builder.SUCCESS
+
+  for failure in sorted(results_tracker.FailedTests()):
+    testabbr = re.sub(r'[^\w\.\-]', '_', failure.split('.')[-1])
+    for line in results_tracker.FailureDescription(failure):
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % (testabbr, line)
+    print '@@@STEP_LOG_END@%s@@@' % testabbr
+
+  for suppression_hash in sorted(results_tracker.SuppressionHashes()):
+    for line in results_tracker.Suppression(suppression_hash):
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % (testabbr, line)
+    print '@@@STEP_LOG_END@%s@@@' % testabbr
+
+  if results_tracker.ParsingErrors():
+    # Generate a log file containing the list of errors.
+    for line in results_tracker.ParsingErrors():
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % ('log parsing error(s)', line)
+
+    print '@@@STEP_LOG_END@%s@@@' % 'log parsing error(s)'
+    results_tracker.ClearParsingErrors()
+
+  if result is builder.SUCCESS:
+    if (len(results_tracker.ParsingErrors()) or
+        len(results_tracker.FailedTests()) or
+        len(results_tracker.SuppressionHashes())):
+      print '@@@STEP_WARNINGS@@@'
+      get_text_result = builder.WARNING
+  else:
+    print '@@@STEP_FAILURE@@@'
+    get_text_result = builder.FAILURE
+
+  for desc in getText(get_text_result, results_tracker, test_name):
+    print '@@@STEP_TEXT@%s@@@' % desc
+
+
 def main_mac(options, args):
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   test_exe = args[0]
   build_dir = os.path.normpath(os.path.abspath(options.build_dir))
   test_exe_path = os.path.join(build_dir, options.target, test_exe)
   if not os.path.exists(test_exe_path):
     pre = 'Unable to find %s\n' % test_exe_path
 
@@ skipping 22 matching lines @@
     command = _BuildParallelCommand(build_dir, test_exe_path, options)
   elif options.run_shell_script:
     command = ['bash', test_exe_path]
   elif options.run_python_script:
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
   command.extend(args[1:])
 
   results_tracker = None
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
+
   if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
-
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('mac', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
     if options.factory_properties.get('asan', False):
       symbolize = os.path.abspath(os.path.join('src', 'tools', 'valgrind',
                                                'asan', 'asan_symbolize.py'))
       pipes = [[sys.executable, symbolize], ['c++filt']]
       result = _RunGTestCommand(command, pipes=pipes)
     else:
       result = _RunGTestCommand(command, results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
+
 def main_linux(options, args):
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   build_dir = os.path.normpath(os.path.abspath(options.build_dir))
   slave_name = slave_utils.SlaveBuildName(build_dir)
   # If this is a sub-project build (i.e. there's a 'sconsbuild' in build_dir),
   # look for the test binaries there, otherwise look for the top-level build
   # output.
   # This assumes we never pass a build_dir which might contain build output that
@@ skipping 63 matching lines @@
     command = _BuildParallelCommand(build_dir, test_exe_path, options)
   elif options.run_shell_script:
     command = ['bash', test_exe_path]
   elif options.run_python_script:
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
   command.extend(args[1:])
 
   results_tracker = None
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
+
   if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
-
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('linux', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
@@ skipping 11 matching lines @@
       result = _RunGTestCommand(command, results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
     if options.xvfb:
       xvfb.StopVirtualX(slave_name)
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
+
 def main_win(options, args):
   """Using the target build configuration, run the executable given in the
   first non-option argument, passing any following arguments to that
   executable.
   """
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   test_exe = args[0]
   build_dir = os.path.abspath(options.build_dir)
@@ skipping 14 matching lines @@
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
   command.extend(args[1:])
 
   # Nuke anything that appears to be stale chrome items in the temporary
   # directory from previous test runs (i.e.- from crashes or unittest leaks).
   slave_utils.RemoveChromeTemporaryFiles()
 
   results_tracker = None
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
+
   if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
-
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('win', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
     result = _RunGTestCommand(command, results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
 
   if options.enable_pageheap:
     slave_utils.SetPageHeap(build_dir, 'chrome.exe', False)
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
 
 def main():
   import platform
 
   xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..',
                            'third_party', 'xvfb', platform.architecture()[0])
 
   # Initialize logging.
@@ skipping 77 matching lines @@
                            help="The name of the builder running this script.")
   option_parser.add_option("", "--build-number", default=None,
                            help=("The build number of the builder running"
                                  "this script."))
   option_parser.add_option("", "--test-type", default='',
                            help="The test name that identifies the test, "
                                 "e.g. 'unit-tests'")
   option_parser.add_option("", "--test-results-server", default='',
                            help="The test results server to upload the "
                                 "results.")
+  option_parser.add_option('', '--annotate', action='store_true',
+                           dest = 'annotate', default=False,
+                           help='Annotate output when run as a buildstep.')
   chromium_utils.AddPropertiesOptions(option_parser)
   options, args = option_parser.parse_args()
 
   if options.run_shell_script and options.run_python_script:
     sys.stderr.write('Use either --run-shell-script OR --run-python-script, '
                      'not both.')
     return 1
 
   # Print out builder name for log_parser
   print '[Running on builder: "%s"]' % options.builder_name
@@ skipping 34 matching lines @@
         '%d new files were left in %s: Fix the tests to clean up themselves.'
         ) % ((new_temp_files - temp_files), tempfile.gettempdir())
     # TODO(maruel): Make it an error soon. Not yet since I want to iron out all
     # the remaining cases before.
     #result = 1
   return result
 
 
 if '__main__' == __name__:
   sys.exit(main())