Convert runtest to have a built-in annotator.

scripts/slave/runtest.py

Index: scripts/slave/runtest.py
diff --git a/scripts/slave/runtest.py b/scripts/slave/runtest.py
index 9be22c5748b7e67d20e9c23c25c0d6d09b98ab1a..33789c89c108b257b622ba29f44612a22df600f2 100755
--- a/scripts/slave/runtest.py
+++ b/scripts/slave/runtest.py
@@ -15,6 +15,7 @@ import copy
 import logging
 import optparse
 import os
+import re
 import stat
 import sys
 import tempfile
@@ -35,7 +36,9 @@ sys.path.insert(0, os.path.abspath('src/tools/python'))
 # pylint checks.
 # pylint: disable=E0611
 # pylint: disable=E1101
+from buildbot.status import builder
 from common import chromium_utils

[cmp, 2012/06/28 19:14:16]

insert an empty line before line 40

+from common import gtest_utils
 from slave import gtest_slave_utils
 from slave import slave_utils
 from slave import xvfb
@@ -80,7 +83,7 @@ def _RunGTestCommand(command, results_tracker=None, pipes=None):
     return 1
   if results_tracker:
     return chromium_utils.RunCommand(
-        command, parser_func=results_tracker.OnReceiveLine)
+        command, parser_func=results_tracker.ProcessLine)
   else:
     return chromium_utils.RunCommand(command, pipes=pipes)
 
@@ -108,7 +111,7 @@ def _GenerateJSONForTestResults(options, results_tracker):
       sys.stderr.write('Unable to generate JSON from XML, using log output.')
       # The file did not get generated. See if we can generate a results map
       # from the log output.
-      results_map = results_tracker.GetResultsMap()
+      results_map = gtest_slave_utils.GetResultsMap(results_tracker)
   except Exception, e:
     # This error will be caught by the following 'not results_map' statement.
     print 'Error: ', e
@@ -189,6 +192,84 @@ def start_http_server(platform, build_dir, test_exe_path, document_root):
                                              (e, output_dir))
   return http_server
 

[cmp, 2012/06/28 19:14:16]

nit: insert an empty line

+def getText(result, observer, name):

[cmp, 2012/06/28 19:14:16]

please add a heredoc with a summary of this method's purpose

[cmp, 2012/06/28 20:59:08]

just checking: is this only used by the annotate() method below?

[Mike Stip (use stip instead), 2012/07/11 00:10:41]

yes, look at getText() in scripts/master/log_parser/gtest_command.py

On 2012/06/28 20:59:08, cmp wrote:

+  GTEST_DASHBOARD_BASE = ("http://test-results.appspot.com"

[cmp, 2012/06/28 20:59:08]

nit: single quotes here and on line 197

+    "/dashboards/flakiness_dashboard.html")
+
+  basic_info = [name]

[cmp, 2012/06/28 20:59:08]

can you add a comment before line 199 describing what basic_info is/how it's
used by buildbot or the annotator?

+
+  disabled = observer.DisabledTests()
+  if disabled:
+    basic_info.append('%s disabled' % str(disabled))
+
+  flaky = observer.FlakyTests()
+  if flaky:
+    basic_info.append('%s flaky' % str(flaky))
+
+  failed_test_count = len(observer.FailedTests())
+
+  if failed_test_count == 0:
+    if result is builder.SUCCESS:
+      return basic_info
+    elif result is builder.WARNINGS:
+      return basic_info + ['warnings']
+
+  if observer.RunningTests():
+    basic_info += ['did not complete']
+
+  if failed_test_count:
+    failure_text = ['failed %d' % failed_test_count]
+    if observer.master_name:
+      # Include the link to the flakiness dashboard
+      failure_text.append('<div class="BuildResultInfo">')
+      failure_text.append('<a href="%s#master=%s&testType=%s'
+                          '&tests=%s">' % (
+          GTEST_DASHBOARD_BASE, observer.master_name,
+          name,
+          ','.join(observer.FailedTests())))
+      failure_text.append('Flakiness dashboard')
+      failure_text.append('</a>')
+      failure_text.append('</div>')
+  else:
+    failure_text = ['crashed or hung']
+  return basic_info + failure_text
+

[cmp, 2012/06/28 19:14:16]

nit: insert an empty line

+def annotate(test_name, result, results_tracker):

[cmp, 2012/06/28 19:14:16]

please add a heredoc with a summary of this method's purpose

+  get_text_result = builder.SUCCESS
+
+  for failure in sorted(results_tracker.FailedTests()):
+    testabbr = re.sub(r'[^\w\.\-]', '_', failure.split('.')[-1])
+    for line in results_tracker.FailureDescription(failure):
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % (testabbr, line)
+    print '@@@STEP_LOG_END@%s@@@' % testabbr
+
+  for suppression_hash in sorted(results_tracker.SuppressionHashes()):
+    for line in results_tracker.Suppression(suppression_hash):
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % (testabbr, line)
+    print '@@@STEP_LOG_END@%s@@@' % testabbr
+
+  if results_tracker.ParsingErrors():
+    # Generate a log file containing the list of errors.
+    for line in results_tracker.ParsingErrors():
+      print '@@@STEP_LOG_LINE@%s@%s@@@' % ('log parsing error(s)', line)
+
+    print '@@@STEP_LOG_END@%s@@@' % 'log parsing error(s)'
+    results_tracker.ClearParsingErrors()
+
+  if result is builder.SUCCESS:
+    if (len(results_tracker.ParsingErrors()) or
+        len(results_tracker.FailedTests()) or
+        len(results_tracker.SuppressionHashes())):
+      print '@@@STEP_WARNINGS@@@'
+      get_text_result = builder.WARNING
+  else:
+    print '@@@STEP_FAILURE@@@'
+    get_text_result = builder.FAILURE
+
+  for desc in getText(get_text_result, results_tracker, test_name):
+    print '@@@STEP_TEXT@%s@@@' % desc
+
+
 def main_mac(options, args):
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
@@ -242,9 +323,10 @@ def main_mac(options, args):
   command.extend(args[1:])
 
   results_tracker = None
-  if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
 
+  if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
@@ -269,6 +351,9 @@ def main_mac(options, args):
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
 def main_linux(options, args):
@@ -365,9 +450,10 @@ def main_linux(options, args):
   command.extend(args[1:])
 
   results_tracker = None
-  if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
 
+  if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
@@ -399,6 +485,9 @@ def main_linux(options, args):
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
 def main_win(options, args):
@@ -446,9 +535,10 @@ def main_win(options, args):
   slave_utils.RemoveChromeTemporaryFiles()
 
   results_tracker = None
-  if options.generate_json_file:
-    results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
+  if options.generate_json_file or options.annotate:
+    results_tracker = gtest_utils.GTestLogParser()
 
+  if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
@@ -470,6 +560,9 @@ def main_win(options, args):
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
+  if options.annotate:
+    annotate(options.test_type, result, results_tracker)
+
   return result
 
 
@@ -567,6 +660,9 @@ def main():
   option_parser.add_option("", "--test-results-server", default='',
                            help="The test results server to upload the "
                                 "results.")
+  option_parser.add_option('', '--annotate', action='store_true',
+                           dest = 'annotate', default=False,
+                           help='Annotate output when run as a buildstep.')
   chromium_utils.AddPropertiesOptions(option_parser)
   options, args = option_parser.parse_args()