Convert runtest to have a built-in annotator.

scripts/slave/runtest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A tool to run a chrome test executable, used by the buildbot slaves.
 
   When this is run, the current directory (cwd) should be the outer build
   directory (e.g., chrome-release/build/).
 
   For a list of command-line options, call this script with '--help'.
 """
 
 import copy
 import logging
 import optparse
 import os
+import re
 import stat
 import sys
 import tempfile
 
 # sys.path needs to be modified here because python2.6 automatically adds the
 # system "google" module (/usr/lib/pymodules/python2.6/google) to sys.modules
 # when we import "chromium_config" (I don't know why it does this). This causes
 # the import of our local "google.*" modules to fail because python seems to
 # only look for a system "google.*", even if our path is in sys.path before
 # importing "google.*". If we modify sys.path here, before importing
 # "chromium_config", python2.6 properly uses our path to find our "google.*"
 # (even though it still automatically adds the system "google" module to
 # sys.modules, and probably should still be using that to resolve "google.*",
 # which I really don't understand).
 sys.path.insert(0, os.path.abspath('src/tools/python'))
 
 # Because of this dependency on a chromium checkout, we need to disable some
 # pylint checks.
 # pylint: disable=E0611
 # pylint: disable=E1101
+from buildbot.status import builder
 from common import chromium_utils

[cmp, 2012/06/28 19:14:16]

insert an empty line before line 40

+from common import gtest_utils
 from slave import gtest_slave_utils
 from slave import slave_utils
 from slave import xvfb
 import config
 
 USAGE = '%s [options] test.exe [test args]' % os.path.basename(sys.argv[0])
 
 CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
 
 DEST_DIR = 'gtest_results'
@@ skipping 24 matching lines @@
   return len(os.listdir(tempfile.gettempdir()))
 
 
 def _RunGTestCommand(command, results_tracker=None, pipes=None):
   if results_tracker and pipes:
     # This is not supported by RunCommand.
     print 'Invalid test invocation. (results_tracker and pipes)'
     return 1
   if results_tracker:
     return chromium_utils.RunCommand(
-        command, parser_func=results_tracker.OnReceiveLine)
+        command, parser_func=results_tracker.ProcessLine)
   else:
     return chromium_utils.RunCommand(command, pipes=pipes)
 
 
 def _GenerateJSONForTestResults(options, results_tracker):
   """Generate (update) a JSON file from the gtest results XML and
   upload the file to the archive server.
   The archived JSON file will be placed at:
   www-dir/DEST_DIR/buildname/testname/results.json
   on the archive server (NOTE: this is to be deprecated).
   Note that it adds slave's WebKit/Tools/Scripts to the PYTHONPATH
   to run the JSON generator.
 
   Args:
     options: command-line options that are supposed to have build_dir,
         results_directory, builder_name, build_name and test_output_xml values.
   """
   # pylint: disable=W0703
   results_map = None
   try:
     if os.path.exists(options.test_output_xml):
       results_map = gtest_slave_utils.GetResultsMapFromXML(
           options.test_output_xml)
     else:
       sys.stderr.write('Unable to generate JSON from XML, using log output.')
       # The file did not get generated. See if we can generate a results map
       # from the log output.
-      results_map = results_tracker.GetResultsMap()
+      results_map = gtest_slave_utils.GetResultsMap(results_tracker)
   except Exception, e:
     # This error will be caught by the following 'not results_map' statement.
     print 'Error: ', e
 
   if not results_map:
     print 'No data was available to update the JSON results'
     return
 
   build_dir = os.path.abspath(options.build_dir)
   slave_name = slave_utils.SlaveBuildName(build_dir)
@@ skipping 59 matching lines @@
                                                  mime_types_path,
                                                  document_root)
   stop_cmd = platform_util.GetStopHttpdCommand()
   http_server = google.httpd_utils.ApacheHttpd(start_cmd, stop_cmd, [8000])
   try:
     http_server.StartServer()
   except google.httpd_utils.HttpdNotStarted, e:
     raise google.httpd_utils.HttpdNotStarted('%s. See log file in %s' %
                                              (e, output_dir))
   return http_server
 

[cmp, 2012/06/28 19:14:16]

nit: insert an empty line

+def getText(result, observer, name):

[cmp, 2012/06/28 19:14:16]

please add a heredoc with a summary of this method's purpose

[cmp, 2012/06/28 20:59:08]

just checking: is this only used by the annotate() method below?

[Mike Stip (use stip instead), 2012/07/11 00:10:41]

yes, look at getText() in scripts/master/log_parser/gtest_command.py

On 2012/06/28 20:59:08, cmp wrote:

+  GTEST_DASHBOARD_BASE = ("http://test-results.appspot.com"

[cmp, 2012/06/28 20:59:08]

nit: single quotes here and on line 197

+    "/dashboards/flakiness_dashboard.html")
+
+  basic_info = [name]

[cmp, 2012/06/28 20:59:08]

can you add a comment before line 199 describing what basic_info is/how it's
used by buildbot or the annotator?

+
+  disabled = observer.DisabledTests()
+  if disabled:
+    basic_info.append('%s disabled' % str(disabled))
+
+  flaky = observer.FlakyTests()
+  if flaky:
+    basic_info.append('%s flaky' % str(flaky))
+
+  failed_test_count = len(observer.FailedTests())
+
+  if failed_test_count == 0:
+    if result is builder.SUCCESS:
+      return basic_info
+    elif result is builder.WARNINGS:
+      return basic_info + ['warnings']
+
+  if observer.RunningTests():
+    basic_info += ['did not complete']
+
+  if failed_test_count:
+    failure_text = ['failed %d' % failed_test_count]
+    if observer.master_name:
+      # Include the link to the flakiness dashboard
+      failure_text.append('<div class="BuildResultInfo">')
+      failure_text.append('<a href="%s#master=%s&testType=%s'
+                          '&tests=%s">' % (
+          GTEST_DASHBOARD_BASE, observer.master_name,
+          name,
+          ','.join(observer.FailedTests())))
+      failure_text.append('Flakiness dashboard')
+      failure_text.append('</a>')
+      failure_text.append('</div>')
+  else:
+    failure_text = ['crashed or hung']
+  return basic_info + failure_text
+

[cmp, 2012/06/28 19:14:16]

nit: insert an empty line

+def annotate(test_name, result, results_tracker):

[cmp, 2012/06/28 19:14:16]

please add a heredoc with a summary of this method's purpose

+  get_text_result = builder.SUCCESS
+
+  for failure in sorted(results_tracker.FailedTests()):
+    testabbr = re.sub(r'[^\w\.\-]', '_', failure.split('.')[-1])
+    for line in results_tracker.FailureDescription(failure):
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % (testabbr, line)
+    print '@@@STEP_LOG_END@%s@@@' % testabbr
+
+  for suppression_hash in sorted(results_tracker.SuppressionHashes()):
+    for line in results_tracker.Suppression(suppression_hash):
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % (testabbr, line)
+    print '@@@STEP_LOG_END@%s@@@' % testabbr
+
+  if results_tracker.ParsingErrors():
+    # Generate a log file containing the list of errors.
+    for line in results_tracker.ParsingErrors():
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % ('log parsing error(s)', line)
+
+    print '@@@STEP_LOG_END@%s@@@' % 'log parsing error(s)'
+    results_tracker.ClearParsingErrors()
+
+  if result is builder.SUCCESS:
+    if (len(results_tracker.ParsingErrors()) or
+        len(results_tracker.FailedTests()) or
+        len(results_tracker.SuppressionHashes())):
+      print '@@@STEP_WARNINGS@@@'
+      get_text_result = builder.WARNING
+  else:
+    print '@@@STEP_FAILURE@@@'
+    get_text_result = builder.FAILURE
+
+  for desc in getText(get_text_result, results_tracker, test_name):
+    print '@@@STEP_TEXT@%s@@@' % desc
+
+
 def main_mac(options, args):
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   test_exe = args[0]
   build_dir = os.path.normpath(os.path.abspath(options.build_dir))
   test_exe_path = os.path.join(build_dir, options.target, test_exe)
   if not os.path.exists(test_exe_path):
     pre = 'Unable to find %s\n' % test_exe_path
 
@@ skipping 33 matching lines @@
     command.append(test_exe_path)
   elif options.run_shell_script:
     command = ['bash', test_exe_path]
   elif options.run_python_script:
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
   command.extend(args[1:])
 
   results_tracker = None
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
+
   if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
-
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('mac', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
     if options.factory_properties.get('asan', False):
       symbolize = os.path.abspath(os.path.join('src', 'tools', 'valgrind',
                                                'asan', 'asan_symbolize.py'))
       pipes = [[sys.executable, symbolize], ['c++filt']]
       result = _RunGTestCommand(command, pipes=pipes)
     else:
       result = _RunGTestCommand(command, results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
 def main_linux(options, args):
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   build_dir = os.path.normpath(os.path.abspath(options.build_dir))
   slave_name = slave_utils.SlaveBuildName(build_dir)
   # If this is a sub-project build (i.e. there's a 'sconsbuild' in build_dir),
   # look for the test binaries there, otherwise look for the top-level build
@@ skipping 76 matching lines @@
     command.append(test_exe_path)
   elif options.run_shell_script:
     command = ['bash', test_exe_path]
   elif options.run_python_script:
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
   command.extend(args[1:])
 
   results_tracker = None
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
+
   if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
-
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('linux', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
@@ skipping 11 matching lines @@
       result = _RunGTestCommand(command, results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
     if options.xvfb:
       xvfb.StopVirtualX(slave_name)
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
 def main_win(options, args):
   """Using the target build configuration, run the executable given in the
   first non-option argument, passing any following arguments to that
   executable.
   """
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
@@ skipping 27 matching lines @@
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
   command.extend(args[1:])
 
   # Nuke anything that appears to be stale chrome items in the temporary
   # directory from previous test runs (i.e.- from crashes or unittest leaks).
   slave_utils.RemoveChromeTemporaryFiles()
 
   results_tracker = None
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
+
   if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
-
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('win', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
     result = _RunGTestCommand(command, results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
 
   if options.enable_pageheap:
     slave_utils.SetPageHeap(build_dir, 'chrome.exe', False)
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
 
 def main():
   import platform
 
   xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..',
                            'third_party', 'xvfb', platform.architecture()[0])
 
   # Initialize logging.
@@ skipping 77 matching lines @@
                            help="The name of the builder running this script.")
   option_parser.add_option("", "--build-number", default=None,
                            help=("The build number of the builder running"
                                  "this script."))
   option_parser.add_option("", "--test-type", default='',
                            help="The test name that identifies the test, "
                                 "e.g. 'unit-tests'")
   option_parser.add_option("", "--test-results-server", default='',
                            help="The test results server to upload the "
                                 "results.")
+  option_parser.add_option('', '--annotate', action='store_true',
+                           dest = 'annotate', default=False,
+                           help='Annotate output when run as a buildstep.')
   chromium_utils.AddPropertiesOptions(option_parser)
   options, args = option_parser.parse_args()
 
   if options.run_shell_script and options.run_python_script:
     sys.stderr.write('Use either --run-shell-script OR --run-python-script, '
                      'not both.')
     return 1
 
   # Print out builder name for log_parser
   print '[Running on builder: "%s"]' % options.builder_name
@@ skipping 34 matching lines @@
         '%d new files were left in %s: Fix the tests to clean up themselves.'
         ) % ((new_temp_files - temp_files), tempfile.gettempdir())
     # TODO(maruel): Make it an error soon. Not yet since I want to iron out all
     # the remaining cases before.
     #result = 1
   return result
 
 
 if '__main__' == __name__:
   sys.exit(main())