Update buildbots to parse new telemetry JSON format.

scripts/slave/results_dashboard.py

Index: scripts/slave/results_dashboard.py
diff --git a/scripts/slave/results_dashboard.py b/scripts/slave/results_dashboard.py
index 7861202c176ca6e9322e616c56ae6bb5b767cb29..a83d6682b4f3bce083019d8f642720c416751c8e 100755
--- a/scripts/slave/results_dashboard.py
+++ b/scripts/slave/results_dashboard.py
@@ -178,7 +178,8 @@ def MakeListOfPoints(charts, bot, test_name, mastername, buildername,
   master = slave_utils.GetActiveMaster()
 
   for chart_name, chart_data in sorted(charts.items()):
-    revision, revision_columns = _RevisionNumberColumns(chart_data, master)
+    revision, default_rev, revision_columns = _RevisionNumberColumns(
+        chart_data, master, 'r_')
 
     for trace_name, trace_values in sorted(chart_data['traces'].items()):
       is_important = trace_name in chart_data.get('important', [])
@@ -193,6 +194,8 @@ def MakeListOfPoints(charts, bot, test_name, mastername, buildername,
           'buildnumber': buildnumber,
           'supplemental_columns': {}
       }
+      if default_rev:
+        supplemental_columns['a_default_rev'] = default_rev
 
       # Add the supplemental_columns values that were passed in after the
       # calculated revision column values so that these can be overwritten.
@@ -226,22 +229,70 @@ def MakeListOfPoints(charts, bot, test_name, mastername, buildername,
   return results
 
 
-def _RevisionNumberColumns(data, master):
+def MakeDashboardJsonV1(chart_json, revision_data, bot, mastername,
+                        buildername, buildnumber, supplemental_dict, is_ref):
+  """Generates Dashboard JSON in the new Telemetry format.
+
+  Args:
+    chart_json: The json output by telemetry
+    revision_data: Data about revisions to include in the upload
+    bot: A string which comes from perf_id, e.g. linux-release.
+    mastername: Buildbot master name, e.g. chromium.perf.
+    buildername: Builder name (for stdio links).
+    buildnumber: Build number (for stdio links).
+    supplemental_columns: A dictionary of extra data to send with a point.
+    is_ref: True if this is a reference build, False otherwise.
+
+  Returns:
+    A dictionary in the format accepted by the perf dashboard.
+  """
+  if not chart_json:
+    print 'Error: No json output from telemetry.'
+    print '@@@STEP_EXCEPTION@@@'
+  # The master name used for the dashboard is the CamelCase name returned by
+  # GetActiveMaster(), and not the canonical master name with dots.
+  master = slave_utils.GetActiveMaster()
+  point_id, default_rev, versions = _RevisionNumberColumns(
+      revision_data, master, '')
+  supplemental_columns = {}
+  if default_rev:
+    supplemental_columns['default_rev'] = default_rev
+  for key in supplemental_dict:
+    supplemental_columns[key.replace('a_', '', 1)] = supplemental_dict[key]
+  fields = {
+      'master': master,
+      'bot': bot,
+      'masterid': mastername,
+      'buildername': buildername,
+      'buildnumber': buildnumber,
+      'point_id': point_id,
+      'supplemental': supplemental_columns,
+      'versions': versions,
+      'chart_data': chart_json,
+      'is_ref': is_ref,
+  }
+  return fields
+
+
+def _GetTimestamp():
+  """Get the Unix timestamp for the current time."""
+  return int(calendar.timegm(datetime.datetime.utcnow().utctimetuple()))
+
+
+def _RevisionNumberColumns(data, master, prefix):
   """Get the revision number and revision-related columns from the given data.
 
   Args:
     data: A dict of information from one line of the log file.
     master: The name of the buildbot master.
+    prefix: Prefix for keys. 'r_' for old-style json, '' for telemetry json.
 
   Returns:
-    A pair with the revision number (which must be an int), and a dict of
-    version-related supplemental columns.
+    A 3-tuple with the revision number (which must be an int), the default rev
+    name, and a dict of version-related supplemental columns.
   """
-  def GetTimestamp():
-    """Get the Unix timestamp for the current time."""
-    return int(calendar.timegm(datetime.datetime.utcnow().utctimetuple()))
-
   revision_supplemental_columns = {}
+  default_rev = None
 
   # The dashboard requires points' x-values to be integers, and points are
   # ordered by this. If the revision can't be parsed as an int, assume that
@@ -252,49 +303,49 @@ def _RevisionNumberColumns(data, master):
   except ValueError:
     # The dashboard requires ordered integer revision numbers. If the revision
     # is not an integer, assume it's a git hash and send a timestamp.
-    revision = GetTimestamp()
+    revision = _GetTimestamp()
     git_hash = data['rev']
 
   # Add Chromium version if it was specified, and use timestamp as x-value.
   if 'ver' in data and data['ver'] != 'undefined':
-    revision_supplemental_columns['r_chrome_version'] = data['ver']
-    revision_supplemental_columns['a_default_rev'] = 'r_chrome_version'
-    revision = GetTimestamp()
+    revision_supplemental_columns[prefix + 'chrome_version'] = data['ver']
+    default_rev = 'r_chrome_version'
+    revision = _GetTimestamp()
 
   # Blink builds can have the same chromium revision for two builds. So
   # order them by timestamp to get them to show on the dashboard in the
   # order they were built.
   if master in ['ChromiumWebkit', 'Oilpan']:
     if not git_hash:
-      revision_supplemental_columns['r_chromium_svn'] = revision
-    revision = GetTimestamp()
+      revision_supplemental_columns[prefix + 'chromium_svn'] = revision
+    revision = _GetTimestamp()
 
   # Regardless of what the master is, if the given "rev" can't be parsed as
   # an int, we're assuming that it's a git hash.
   if git_hash:
-    revision_supplemental_columns['r_chromium'] = git_hash
+    revision_supplemental_columns[prefix + 'chromium'] = git_hash
 
   # For Oilpan, send the webkit_rev as r_oilpan since we are getting
   # the oilpan branch revision instead of the Blink trunk revision
   # and set r_oilpan to be the dashboard default revision.
   if master == 'Oilpan':
-    revision_supplemental_columns['r_oilpan'] = data['webkit_rev']
-    revision_supplemental_columns['a_default_rev'] = 'r_oilpan'
+    revision_supplemental_columns[prefix + 'oilpan'] = data['webkit_rev']
+    default_rev = 'r_oilpan'
   else:
     # For other revision data, add it if it's present and not undefined:
     for key in ['webkit_rev', 'webrtc_rev', 'v8_rev']:
       if key in data and data[key] != 'undefined':
-        revision_supplemental_columns['r_' + key] = data[key]
+        revision_supplemental_columns[prefix + key] = data[key]
 
   # If possible, also send the git hash.
   # If no other "default revision" type is specified already, use the git hash.
   # This will change how it is displayed on the perf dashboard.
   if 'git_revision' in data and data['git_revision'] != 'undefined':
-    revision_supplemental_columns['r_chromium'] = data['git_revision']
-    if 'a_default_rev' not in revision_supplemental_columns:
-      revision_supplemental_columns['a_default_rev'] = 'r_chromium'
+    revision_supplemental_columns[prefix + 'chromium'] = data['git_revision']
+    if not default_rev:
+      default_rev = 'r_chromium'
 
-  return revision, revision_supplemental_columns
+  return revision, default_rev, revision_supplemental_columns
 
 
 def _TestPath(test_name, chart_name, trace_name):
@@ -367,12 +418,16 @@ def _LinkAnnotation(url, data):
   Returns:
     An annotation to print, or None.
   """
-  if not data or type(data) is not list:
+  if not data:
     return None
-  point = data[0]
+  if type(data) is list:

[ghost stip (do not use), 2014/09/12 18:17:25]

usually better to do

if isinstance(data, list):

[sullivan, 2014/09/12 19:14:21]

Done.

+    master, bot, test, revision = (
+        data[0]['master'], data[0]['bot'], data[0]['test'], data[0]['revision'])
+  else:
+    master, bot, test, revision = (
+        data['master'], data['bot'], data['chart_data']['benchmark_name'],
+        data['point_id'])
   results_link = url + RESULTS_LINK_PATH % (
-      urllib.quote(point['master']),
-      urllib.quote(point['bot']),
-      urllib.quote(point['test'].split('/')[0]),
-      point['revision'])
+      urllib.quote(master), urllib.quote(bot), urllib.quote(test.split('/')[0]),
+      revision)
   return '@@@STEP_LINK@%s@%s@@@' % ('Results Dashboard', results_link)