Update buildbots to parse new telemetry JSON format.

scripts/slave/results_dashboard.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Functions for adding results to perf dashboard."""
 
 import calendar
 import datetime
 import httplib
@@ skipping 160 matching lines @@
     Each dictionary has the keys "master", "bot", "test", "value", "revision".
     The full details of this format are described at http://goo.gl/TcJliv.
   """
   results = []
 
   # The master name used for the dashboard is the CamelCase name returned by
   # GetActiveMaster(), and not the canonical master name with dots.
   master = slave_utils.GetActiveMaster()
 
   for chart_name, chart_data in sorted(charts.items()):
-    revision, revision_columns = _RevisionNumberColumns(chart_data, master)
+    revision, default_rev, revision_columns = _RevisionNumberColumns(
+        chart_data, master, 'r_')
 
     for trace_name, trace_values in sorted(chart_data['traces'].items()):
       is_important = trace_name in chart_data.get('important', [])
       test_path = _TestPath(test_name, chart_name, trace_name)
       result = {
           'master': master,
           'bot': bot,
           'test': test_path,
           'revision': revision,
           'masterid': mastername,
           'buildername': buildername,
           'buildnumber': buildnumber,
           'supplemental_columns': {}
       }
+      if default_rev:
+        supplemental_columns['a_default_rev'] = default_rev
 
       # Add the supplemental_columns values that were passed in after the
       # calculated revision column values so that these can be overwritten.
       result['supplemental_columns'].update(revision_columns)
       result['supplemental_columns'].update(supplemental_columns)
 
       # Check whether this result is itself a time series (e.g. Endure result).
       # If so add the key "data", otherwise add "value" and "error".
       have_multi_value_data = False
       for value in trace_values:
@@ skipping 13 matching lines @@
       if chart_data.get('stack'):
         result['stack'] = chart_data['stack']
       if is_important:
         result['important'] = True
 
       results.append(result)
 
   return results
 
 
-def _RevisionNumberColumns(data, master):
+def MakeDashboardJsonV1(chart_json, revision_data, bot, mastername,
+                        buildername, buildnumber, supplemental_dict, is_ref):
+  """Generates Dashboard JSON in the new Telemetry format.
+
+  Args:
+    chart_json: The json output by telemetry
+    revision_data: Data about revisions to include in the upload
+    bot: A string which comes from perf_id, e.g. linux-release.
+    mastername: Buildbot master name, e.g. chromium.perf.
+    buildername: Builder name (for stdio links).
+    buildnumber: Build number (for stdio links).
+    supplemental_columns: A dictionary of extra data to send with a point.
+    is_ref: True if this is a reference build, False otherwise.
+
+  Returns:
+    A dictionary in the format accepted by the perf dashboard.
+  """
+  if not chart_json:
+    print 'Error: No json output from telemetry.'
+    print '@@@STEP_EXCEPTION@@@'
+  # The master name used for the dashboard is the CamelCase name returned by
+  # GetActiveMaster(), and not the canonical master name with dots.
+  master = slave_utils.GetActiveMaster()
+  point_id, default_rev, versions = _RevisionNumberColumns(
+      revision_data, master, '')
+  supplemental_columns = {}
+  if default_rev:
+    supplemental_columns['default_rev'] = default_rev
+  for key in supplemental_dict:
+    supplemental_columns[key.replace('a_', '', 1)] = supplemental_dict[key]
+  fields = {
+      'master': master,
+      'bot': bot,
+      'masterid': mastername,
+      'buildername': buildername,
+      'buildnumber': buildnumber,
+      'point_id': point_id,
+      'supplemental': supplemental_columns,
+      'versions': versions,
+      'chart_data': chart_json,
+      'is_ref': is_ref,
+  }
+  return fields
+
+
+def _GetTimestamp():
+  """Get the Unix timestamp for the current time."""
+  return int(calendar.timegm(datetime.datetime.utcnow().utctimetuple()))
+
+
+def _RevisionNumberColumns(data, master, prefix):
   """Get the revision number and revision-related columns from the given data.
 
   Args:
     data: A dict of information from one line of the log file.
     master: The name of the buildbot master.
+    prefix: Prefix for keys. 'r_' for old-style json, '' for telemetry json.
 
   Returns:
-    A pair with the revision number (which must be an int), and a dict of
-    version-related supplemental columns.
+    A 3-tuple with the revision number (which must be an int), the default rev
+    name, and a dict of version-related supplemental columns.
   """
-  def GetTimestamp():
-    """Get the Unix timestamp for the current time."""
-    return int(calendar.timegm(datetime.datetime.utcnow().utctimetuple()))
-
   revision_supplemental_columns = {}
+  default_rev = None
 
   # The dashboard requires points' x-values to be integers, and points are
   # ordered by this. If the revision can't be parsed as an int, assume that
   # it's a git hash and use timestamp as the x-value.
   git_hash = None
   try:
     revision = int(data['rev'])
   except ValueError:
     # The dashboard requires ordered integer revision numbers. If the revision
     # is not an integer, assume it's a git hash and send a timestamp.
-    revision = GetTimestamp()
+    revision = _GetTimestamp()
     git_hash = data['rev']
 
   # Add Chromium version if it was specified, and use timestamp as x-value.
   if 'ver' in data and data['ver'] != 'undefined':
-    revision_supplemental_columns['r_chrome_version'] = data['ver']
-    revision_supplemental_columns['a_default_rev'] = 'r_chrome_version'
-    revision = GetTimestamp()
+    revision_supplemental_columns[prefix + 'chrome_version'] = data['ver']
+    default_rev = 'r_chrome_version'
+    revision = _GetTimestamp()
 
   # Blink builds can have the same chromium revision for two builds. So
   # order them by timestamp to get them to show on the dashboard in the
   # order they were built.
   if master in ['ChromiumWebkit', 'Oilpan']:
     if not git_hash:
-      revision_supplemental_columns['r_chromium_svn'] = revision
-    revision = GetTimestamp()
+      revision_supplemental_columns[prefix + 'chromium_svn'] = revision
+    revision = _GetTimestamp()
 
   # Regardless of what the master is, if the given "rev" can't be parsed as
   # an int, we're assuming that it's a git hash.
   if git_hash:
-    revision_supplemental_columns['r_chromium'] = git_hash
+    revision_supplemental_columns[prefix + 'chromium'] = git_hash
 
   # For Oilpan, send the webkit_rev as r_oilpan since we are getting
   # the oilpan branch revision instead of the Blink trunk revision
   # and set r_oilpan to be the dashboard default revision.
   if master == 'Oilpan':
-    revision_supplemental_columns['r_oilpan'] = data['webkit_rev']
-    revision_supplemental_columns['a_default_rev'] = 'r_oilpan'
+    revision_supplemental_columns[prefix + 'oilpan'] = data['webkit_rev']
+    default_rev = 'r_oilpan'
   else:
     # For other revision data, add it if it's present and not undefined:
     for key in ['webkit_rev', 'webrtc_rev', 'v8_rev']:
       if key in data and data[key] != 'undefined':
-        revision_supplemental_columns['r_' + key] = data[key]
+        revision_supplemental_columns[prefix + key] = data[key]
 
   # If possible, also send the git hash.
   # If no other "default revision" type is specified already, use the git hash.
   # This will change how it is displayed on the perf dashboard.
   if 'git_revision' in data and data['git_revision'] != 'undefined':
-    revision_supplemental_columns['r_chromium'] = data['git_revision']
-    if 'a_default_rev' not in revision_supplemental_columns:
-      revision_supplemental_columns['a_default_rev'] = 'r_chromium'
+    revision_supplemental_columns[prefix + 'chromium'] = data['git_revision']
+    if not default_rev:
+      default_rev = 'r_chromium'
 
-  return revision, revision_supplemental_columns
+  return revision, default_rev, revision_supplemental_columns
 
 
 def _TestPath(test_name, chart_name, trace_name):
   """Get the slash-separated test path to send.
 
   Args:
     test: Test name. Typically, this will be a top-level 'test suite' name.
     chart_name: Name of a chart where multiple trace lines are grouped. If the
         chart name is the same as the trace name, that signifies that this is
         the main trace for the chart.
@@ skipping 52 matching lines @@
 def _LinkAnnotation(url, data):
   """Prints a link annotation with a link to the dashboard if possible.
 
   Args:
     url: The Performance Dashboard URL, e.g. "https://chromeperf.appspot.com"
     data: The data that's being sent to the dashboard.
 
   Returns:
     An annotation to print, or None.
   """
-  if not data or type(data) is not list:
+  if not data:
     return None
-  point = data[0]
+  if isinstance(data, list):
+    master, bot, test, revision = (
+        data[0]['master'], data[0]['bot'], data[0]['test'], data[0]['revision'])
+  else:
+    master, bot, test, revision = (
+        data['master'], data['bot'], data['chart_data']['benchmark_name'],
+        data['point_id'])
   results_link = url + RESULTS_LINK_PATH % (
-      urllib.quote(point['master']),
-      urllib.quote(point['bot']),
-      urllib.quote(point['test'].split('/')[0]),
-      point['revision'])
+      urllib.quote(master), urllib.quote(bot), urllib.quote(test.split('/')[0]),
+      revision)
   return '@@@STEP_LINK@%s@%s@@@' % ('Results Dashboard', results_link)