GTTF: Make runtest.py use the new test launcher's JSON summary.

scripts/slave/runtest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A tool to run a chrome test executable, used by the buildbot slaves.
 
 When this is run, the current directory (cwd) should be the outer build
 directory (e.g., chrome-release/build/).
 
@@ skipping 170 matching lines @@
   Note that it adds slave's WebKit/Tools/Scripts to the PYTHONPATH
   to run the JSON generator.
 
   Args:
     options: command-line options that are supposed to have build_dir,
         results_directory, builder_name, build_name and test_output_xml values.
   """
   # pylint: disable=W0703
   results_map = None
   try:
-    if os.path.exists(options.test_output_xml):
+    if (os.path.exists(options.test_output_xml) and
+        not using_gtest_json(options)):
       results_map = gtest_slave_utils.GetResultsMapFromXML(
           options.test_output_xml)
     else:
-      sys.stderr.write(
-          ('"%s" \ "%s" doesn\'t exist: Unable to generate JSON from XML, '
-           'using log output.\n') % (os.getcwd(), options.test_output_xml))
+      if using_gtest_json(options):

[ghost stip (do not use), 2013/12/16 21:18:50]

don't you still need the XML reader here for the flakiness dashboard?

[Paweł Hajdan Jr., 2013/12/17 16:51:56]

Sorry, could you explain more? We discussed this offline - data for
test-results.appspot.com is generated based on the JSON summary, not on the
gtest XML file we don't have here.

[ghost stip (do not use), 2013/12/17 18:36:39]

ah sorry, I misread the if indentation and thought you weren't setting
results_map. it looks like line 204 fires though

+        sys.stderr.write('using JSON summary output instead of gtest XML\n')
+      else:
+        sys.stderr.write(
+            ('"%s" \ "%s" doesn\'t exist: Unable to generate JSON from XML, '
+             'using log output.\n') % (os.getcwd(), options.test_output_xml))
       # The file did not get generated. See if we can generate a results map
       # from the log output.
       results_map = gtest_slave_utils.GetResultsMap(results_tracker)
   except Exception, e:
     # This error will be caught by the following 'not results_map' statement.
     print 'Error: ', e
 
   if not results_map:
     print 'No data was available to update the JSON results'
     return
@@ skipping 122 matching lines @@
   stop_cmd = platform_util.GetStopHttpdCommand()
   http_server = google.httpd_utils.ApacheHttpd(start_cmd, stop_cmd, [8000])
   try:
     http_server.StartServer()
   except google.httpd_utils.HttpdNotStarted, e:
     raise google.httpd_utils.HttpdNotStarted('%s. See log file in %s' %
                                              (e, output_dir))
   return http_server
 
 
+def using_gtest_json(options):
+  """Returns true if we're using gtest JSON summary."""
+  return (options.annotate == 'gtest' and
+          not options.run_python_script and
+          not options.run_shell_script)
+
+
 def get_parsers():
   """Returns a dictionary mapping strings to log parser classes."""
   parsers = {'gtest': gtest_utils.GTestLogParser,
              'benchpress': process_log_utils.BenchpressLogProcessor,
              'playback': process_log_utils.PlaybackLogProcessor,
              'graphing': process_log_utils.GraphingLogProcessor,
              'endure': process_log_utils.GraphingEndureLogProcessor,
              'framerate': process_log_utils.GraphingFrameRateLogProcessor,
              'pagecycler': process_log_utils.GraphingPageCyclerLogProcessor}
   return parsers
 
 
 def list_parsers(selection):
   """Prints a list of available log parser classes iff the input is 'list'.
 
   Returns:
     True iff the input is 'list' (meaning that a list was printed).
   """
   parsers = get_parsers()
   shouldlist = selection and selection == 'list'
   if shouldlist:
     print
     print 'Available log parsers:'
     for p in parsers:
       print ' ', p, parsers[p].__name__
 
   return shouldlist
 
 
-def select_results_tracker(selection, use_gtest):
+def select_results_tracker(options):
   """Returns a log parser class (aka results tracker class).
 
   Args:
-    selection: Name of parser to use. This is the value of the
-        of the --annotate option that's passed to this script.
-    use_gtest: Whether the gtest parser should be used. This is
-        the value of the --generate-json-file flag.
+    options: Command-line options (from OptionParser).
 
   Returns:
     A log parser class (aka results tracker class), or None.
   """
+  if (using_gtest_json(options)):
+    return gtest_utils.GTestJSONParser
+
   parsers = get_parsers()
-  if selection:
-    if selection in parsers:
-      if use_gtest and selection != 'gtest':
+  if options.annotate:
+    if options.annotate in parsers:
+      if options.generate_json_file and options.annotate != 'gtest':
         raise NotImplementedError("'%s' doesn't make sense with "
                                   "options.generate_json_file")
       else:
-        return parsers[selection]
+        return parsers[options.annotate]
     else:
-      raise KeyError("'%s' is not a valid GTest parser!!" % selection)
-  elif use_gtest:
+      raise KeyError("'%s' is not a valid GTest parser!!" % options.annotate)
+  elif options.generate_json_file:
     return parsers['gtest']
+
   return None
 
 
 def create_results_tracker(tracker_class, options):
   """Instantiate a log parser (aka results tracker).
 
   Args:
     tracker_class: A log parser class.
     options: Command-line options (from OptionParser).
 
   Returns:
     An instance of a log parser class, or None.
   """
   if not tracker_class:
     return None
 
-  if tracker_class.__name__ == 'GTestLogParser':
+  if tracker_class.__name__ in ('GTestLogParser', 'GTestJSONParser'):
     tracker_obj = tracker_class()
   else:
     build_dir = os.path.abspath(options.build_dir)
     try:
       webkit_dir = chromium_utils.FindUpward(build_dir, 'third_party', 'WebKit',
                                              'Source')
       webkit_revision = GetSvnRevision(webkit_dir)
     except:  # pylint: disable=W0702
       webkit_revision = 'undefined'
 
@@ skipping 168 matching lines @@
   """
 
   if not options.annotate:
     raise chromium_utils.MissingArgument('--parse-input doesn\'t make sense '
                                          'without --annotate.')
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if list_parsers(options.annotate):
     return 0
 
-  tracker_class = select_results_tracker(options.annotate,
-                                         options.generate_json_file)
+  tracker_class = select_results_tracker(options)
   results_tracker = create_results_tracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   if options.parse_input == '-':
     f = sys.stdin
   else:
@@ skipping 44 matching lines @@
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
     if options.annotate == 'gtest':
       command.extend(['--brave-new-test-launcher', '--test-launcher-bot-mode'])
   command.extend(args[1:])
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if list_parsers(options.annotate):
     return 0
-  tracker_class = select_results_tracker(options.annotate,
-                                         options.generate_json_file)
+  tracker_class = select_results_tracker(options)
   results_tracker = create_results_tracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('mac', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
+
+    if using_gtest_json(options):
+      json_file_name = results_tracker.OpenJSONFile(
+          options.test_launcher_summary_output)
+      command.append('--test-launcher-summary-output=%s' % json_file_name)
+
     pipes = []
     if options.factory_properties.get('asan', False):
       symbolize = os.path.abspath(os.path.join('src', 'tools', 'valgrind',
                                                'asan', 'asan_symbolize.py'))
       pipes = [[sys.executable, symbolize], ['c++filt']]
 
     command = generate_run_isolated_command(build_dir, test_exe_path, options,
                                             command)
     result = _RunGTestCommand(command, pipes=pipes,
                               results_tracker=results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
+    if using_gtest_json(options):
+      results_tracker.ProcessAndCloseJSONFile()
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
   if options.annotate:
     annotation_utils.annotate(
         options.test_type, result, results_tracker,
         options.factory_properties.get('full_test_name'),
         perf_dashboard_id=options.perf_dashboard_id)
 
@@ skipping 194 matching lines @@
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
     if options.annotate == 'gtest':
       command.extend(['--brave-new-test-launcher', '--test-launcher-bot-mode'])
   command.extend(args[1:])
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if list_parsers(options.annotate):
     return 0
-  tracker_class = select_results_tracker(options.annotate,
-                                         options.generate_json_file)
+  tracker_class = select_results_tracker(options)
   results_tracker = create_results_tracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     start_xvfb = False
     http_server = None
     if options.document_root:
       http_server = start_http_server('linux', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
 
     # TODO(dpranke): checking on test_exe is a temporary hack until we
     # can change the buildbot master to pass --xvfb instead of --no-xvfb
     # for these two steps. See
     # https://code.google.com/p/chromium/issues/detail?id=179814
     start_xvfb = (options.xvfb or
                   'layout_test_wrapper' in test_exe or
                   'devtools_perf_test_wrapper' in test_exe)
     if start_xvfb:
       xvfb.StartVirtualX(
           slave_name, bin_dir,
           with_wm=(options.factory_properties.get('window_manager', 'True') ==
                    'True'),
           server_dir=special_xvfb_dir)
 
+    if using_gtest_json(options):
+      json_file_name = results_tracker.OpenJSONFile(
+          options.test_launcher_summary_output)
+      command.append('--test-launcher-summary-output=%s' % json_file_name)
+
     pipes = []
     # Plain ASan bots use a symbolizer script, whereas ASan+LSan and LSan bots
     # use a built-in symbolizer.
     if (options.factory_properties.get('asan', False) and
         not options.enable_lsan):
       symbolize = os.path.abspath(os.path.join('src', 'tools', 'valgrind',
                                                'asan', 'asan_symbolize.py'))
       pipes = [[sys.executable, symbolize], ['c++filt']]
 
     command = generate_run_isolated_command(build_dir, test_exe_path, options,
                                             command)
     result = _RunGTestCommand(command, pipes=pipes,
                               results_tracker=results_tracker,
                               extra_env=extra_env)
   finally:
     if http_server:
       http_server.StopServer()
     if start_xvfb:
       xvfb.StopVirtualX(slave_name)
+    if using_gtest_json(options):
+      results_tracker.ProcessAndCloseJSONFile()
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
   if options.annotate:
     annotation_utils.annotate(
         options.test_type, result, results_tracker,
         options.factory_properties.get('full_test_name'),
         perf_dashboard_id=options.perf_dashboard_id)
 
@@ skipping 59 matching lines @@
                '--'] + command
   command.extend(args[1:])
 
   # Nuke anything that appears to be stale chrome items in the temporary
   # directory from previous test runs (i.e.- from crashes or unittest leaks).
   slave_utils.RemoveChromeTemporaryFiles()
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if list_parsers(options.annotate):
     return 0
-  tracker_class = select_results_tracker(options.annotate,
-                                         options.generate_json_file)
+  tracker_class = select_results_tracker(options)
   results_tracker = create_results_tracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = start_http_server('win', build_dir=build_dir,
                                       test_exe_path=test_exe_path,
                                       document_root=options.document_root)
+
+    if using_gtest_json(options):
+      json_file_name = results_tracker.OpenJSONFile(
+          options.test_launcher_summary_output)
+      command.append('--test-launcher-summary-output=%s' % json_file_name)
+
     command = generate_run_isolated_command(build_dir, test_exe_path, options,
                                             command)
     result = _RunGTestCommand(command, results_tracker)
   finally:
     if http_server:
       http_server.StopServer()
+    if using_gtest_json(options):
+      results_tracker.ProcessAndCloseJSONFile()
 
   if options.enable_pageheap:
     slave_utils.SetPageHeap(build_dir, 'chrome.exe', False)
 
   if options.generate_json_file:
     _GenerateJSONForTestResults(options, results_tracker)
 
   if options.annotate:
     annotation_utils.annotate(
         options.test_type, result, results_tracker,
@@ skipping 21 matching lines @@
   Python scripts are the same as with linux.
   """
   if options.run_python_script:
     return main_linux(options, args)
 
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   if list_parsers(options.annotate):
     return 0
-  tracker_class = select_results_tracker(options.annotate,
-                                         options.generate_json_file)
+  tracker_class = select_results_tracker(options)
   results_tracker = create_results_tracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   # Assume it's a gtest apk, so use the android harness.
   test_suite = args[0]
   run_test_target_option = '--release'
@@ skipping 157 matching lines @@
   option_parser.add_option('--enable-lsan', default=False,
                            help='Enable memory leak detection (LeakSanitizer). '
                                 'Also can be enabled with the factory '
                                 'properties "lsan" and "lsan_run_all_tests".')
   option_parser.add_option('--extra-sharding-args', default='',
                            help='Extra options for run_test_cases.py.')
   option_parser.add_option('--no-spawn-dbus', action='store_true',
                            default=False,
                            help='Disable GLib DBus bug workaround: '
                                 'manually spawning dbus-launch')
+  option_parser.add_option('--test-launcher-summary-output',
+                           help='Path to test results file with all the info '
+                                'from the test launcher')
 
   chromium_utils.AddPropertiesOptions(option_parser)
   options, args = option_parser.parse_args()
   if not options.perf_dashboard_id:
     options.perf_dashboard_id = options.factory_properties.get('test_name')
 
   options.test_type = options.test_type or options.factory_properties.get(
       'step_name', '')
 
   if options.run_shell_script and options.run_python_script:
@@ skipping 156 matching lines @@
     return result
   finally:
     if did_launch_dbus:
       # It looks like the command line argument --exit-with-session
       # isn't working to clean up the spawned dbus-daemon. Kill it
       # manually.
       _ShutdownDBus()
 
 if '__main__' == __name__:
   sys.exit(main())