Update buildbots to parse new telemetry JSON format.

scripts/slave/runtest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A tool used to run a Chrome test executable and process the output.
 
 This script is used by the buildbot slaves. It must be run from the outer
 build directory, e.g. chrome-release/build/.
 
@@ skipping 36 matching lines @@
 # TODO(crbug.com/403564). We almost certainly shouldn't be importing this.
 import config
 
 from slave import annotation_utils
 from slave import build_directory
 from slave import crash_utils
 from slave import gtest_slave_utils
 from slave import process_log_utils
 from slave import results_dashboard
 from slave import slave_utils
+from slave import telemetry_utils
 from slave import xvfb
 
 USAGE = '%s [options] test.exe [test args]' % os.path.basename(sys.argv[0])
 
 CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
 
 # Directory to write JSON for test results into.
 DEST_DIR = 'gtest_results'
 
 # Names of httpd configuration file under different platforms.
@@ skipping 479 matching lines @@
   shouldlist = selection and selection == 'list'
   if shouldlist:
     print
     print 'Available log parsers:'
     for p in parsers:
       print ' ', p, parsers[p].__name__
 
   return shouldlist
 
 
-def _SelectResultsTracker(options):
+def _SelectResultsTracker(options, test_exe):
   """Returns a log parser class (aka results tracker class).
 
   Args:
     options: Command-line options (from OptionParser).
+    test_exe: Name of the test to execute
 
   Returns:
     A log parser class (aka results tracker class), or None.
   """
   if _UsingGtestJson(options):
     return gtest_utils.GTestJSONParser
 
+  if test_exe and test_exe.endswith('telemetry.py'):
+    return telemetry_utils.TelemetryLogParser

[sullivan, 2014/09/05 04:30:00]

Is this the best way to tell if it's a telemetry test?

[ghost stip (do not use), 2014/09/05 21:39:06]

Unfortunately yeah, if you want skip a master restart this is the best way.

+
   parsers = _GetParsers()
   if options.annotate:
     if options.annotate in parsers:
       if options.generate_json_file and options.annotate != 'gtest':
         raise NotImplementedError('"%s" doesn\'t make sense with '
                                   'options.generate_json_file.')
       else:
         return parsers[options.annotate]
     else:
       raise KeyError('"%s" is not a valid GTest parser!' % options.annotate)
@@ skipping 17 matching lines @@
   Args:
     tracker_class: A log parser class.
     options: Command-line options (from OptionParser).
 
   Returns:
     An instance of a log parser class, or None.
   """
   if not tracker_class:
     return None
 
-  if tracker_class.__name__ in ('GTestLogParser',):
+  if tracker_class.__name__ in ('GTestLogParser', 'TelemetryLogParser'):
     tracker_obj = tracker_class()
   elif tracker_class.__name__ in ('GTestJSONParser',):
     tracker_obj = tracker_class(options.build_properties.get('mastername'))
   else:
     build_dir = os.path.abspath(options.build_dir)
 
     if options.webkit_revision:
       webkit_revision = options.webkit_revision
     else:
       try:
@@ skipping 67 matching lines @@
   """
   if system is None:
     # perf_id not specified in factory properties.
     print 'Error: No system name (perf_id) specified when sending to dashboard.'
     return
   supplemental_columns = _GetSupplementalColumns(
       build_dir, supplemental_columns_file)
   if extra_columns:
     supplemental_columns.update(extra_columns)
 
-  charts = _GetDataFromLogProcessor(results_tracker)
-  points = results_dashboard.MakeListOfPoints(
-      charts, system, test, mastername, buildername, buildnumber,
-      supplemental_columns)
-  results_dashboard.SendResults(points, url, build_dir)
+  if results_tracker.IsChartJson():
+    results_dashboard.SendChartJsonResults(

[sullivan, 2014/09/05 04:30:00]

Still working on implementing this.

[sullivan, 2014/09/05 21:32:16]

This is implemented now.

+        results_tracker.Chart(), results_tracker.RefChart(),
+        system, test, mastername, buildername, buildnumber, supplemental_columns
+    )
+  else:
+    charts = _GetDataFromLogProcessor(results_tracker)
+    points = results_dashboard.MakeListOfPoints(
+        charts, system, test, mastername, buildername, buildnumber,
+        supplemental_columns)
+    results_dashboard.SendResults(points, url, build_dir)
 
 
 def _GetDataFromLogProcessor(log_processor):
   """Returns a mapping of chart names to chart data.
 
   Args:
     log_processor: A log processor (aka results tracker) object.
 
   Returns:
     A dictionary mapping chart name to lists of chart data.
@@ skipping 247 matching lines @@
   through the specified annotation parser.
   """
   if not options.annotate:
     raise chromium_utils.MissingArgument('--parse-input doesn\'t make sense '
                                          'without --annotate.')
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if _ListParsers(options.annotate):
     return 0
 
-  tracker_class = _SelectResultsTracker(options)
+  tracker_class = _SelectResultsTracker(options, None)

[sullivan, 2014/09/05 04:30:00]

This should never get called for Telemetry tests, right?

[ghost stip (do not use), 2014/09/05 21:39:06]

Right, there is no binary here

   results_tracker = _CreateResultsTracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   if options.parse_input == '-':
     f = sys.stdin
   else:
@@ skipping 46 matching lines @@
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
     if options.annotate == 'gtest':
       command.extend(['--brave-new-test-launcher', '--test-launcher-bot-mode'])
   command.extend(args[1:])
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if _ListParsers(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
+  tracker_class = _SelectResultsTracker(options, test_exe)
   results_tracker = _CreateResultsTracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
@@ skipping 41 matching lines @@
         options.test_type, options.results_url, options.build_dir,
         options.build_properties.get('mastername'),
         options.build_properties.get('buildername'),
         options.build_properties.get('buildnumber'),
         options.supplemental_columns_file,
         options.perf_config)
 
   return result
 
 
 def _MainIOS(options, args, extra_env):

[sullivan, 2014/09/05 04:30:00]

Is telemetry available on iOS? Not sure how perf test result parsing works on
iOS now.

[ghost stip (do not use), 2014/09/05 21:39:06]

I'm not sure. smut@?

[smut, 2014/09/05 21:59:09]

Perf tests only run downstream on iOS, and downstream doesn't invoke runtest.py
at all. Telemetry tests don't currently run on iOS up- or downstream, but
baxley@ is working on it.

If we get telemetry tests running upstream, we'll probably need support here,
and for downstream I can point you to where the TelemetryLogParser can be
integrated.

   """Runs the test on iOS."""
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   def kill_simulator():
     chromium_utils.RunCommand(['/usr/bin/killall', 'iPhone Simulator'])
 
   # For iOS tests, the args come in in the following order:
   #   [0] test display name formatted as 'test_name (device[ ios_version])'
   #   [1:] gtest args (e.g. --gtest_print_time)
@@ skipping 170 matching lines @@
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
     if options.annotate == 'gtest':
       command.extend(['--brave-new-test-launcher', '--test-launcher-bot-mode'])
   command.extend(args[1:])
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if _ListParsers(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
+  tracker_class = _SelectResultsTracker(options, test_exe)
   results_tracker = _CreateResultsTracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     start_xvfb = False
     http_server = None
@@ skipping 128 matching lines @@
                '--'] + command
   command.extend(args[1:])
 
   # Nuke anything that appears to be stale chrome items in the temporary
   # directory from previous test runs (i.e.- from crashes or unittest leaks).
   slave_utils.RemoveChromeTemporaryFiles()
 
   # If --annotate=list was passed, list the log parser classes and exit.
   if _ListParsers(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
+  tracker_class = _SelectResultsTracker(options, test_exe)
   results_tracker = _CreateResultsTracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
@@ skipping 37 matching lines @@
         options.test_type, options.results_url, options.build_dir,
         options.build_properties.get('mastername'),
         options.build_properties.get('buildername'),
         options.build_properties.get('buildnumber'),
         options.supplemental_columns_file,
         options.perf_config)
 
   return result
 
 
 def _MainAndroid(options, args, extra_env):

[sullivan, 2014/09/05 04:30:00]

The command line looks different for Android, I think because recipes has been
enabled on Android:

http://chromegw/i/chromium.perf/builders/Android%20Nexus5%20Perf/builds/1580/...

Do you know any way to check if we should use the TelemetryLogParser here?
Ideally we'd do something future-proof for when the other platforms switch over
to recipes.

[ghost stip (do not use), 2014/09/05 21:39:06]

Android uses a far different way of invoking tests, because they have their own
custom runner script. I'm looping in luqui@ who might be able to help.

   """Runs tests on android.
 
   Running GTest-based tests on android is different than on Linux as it requires
   src/build/android/test_runner.py to deploy and communicate with the device.
   Python scripts are the same as with Linux.
 
   Args:
     options: Command-line options for this invocation of runtest.py.
     args: Command and arguments for the test.
     extra_env: A dictionary of extra environment variables to set.
 
   Returns:
     Exit status code.
   """
   if options.run_python_script:
     return _MainLinux(options, args, extra_env)
 
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
   if _ListParsers(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
+  tracker_class = _SelectResultsTracker(options, None)
   results_tracker = _CreateResultsTracker(tracker_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   # Assume it's a gtest apk, so use the android harness.
   test_suite = args[0]
   run_test_target_option = '--release'
@@ skipping 423 matching lines @@
   finally:
     if did_launch_dbus:
       # It looks like the command line argument --exit-with-session
       # isn't working to clean up the spawned dbus-daemon. Kill it
       # manually.
       _ShutdownDBus()
 
 
 if '__main__' == __name__:
   sys.exit(main())