BuildBucket-based build triggering

scripts/master/chromium_step.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Subclasses of various slave command classes."""
 
 from datetime import datetime
 import copy
 import errno
 import json
 import logging
 import os
 import time
 
 from twisted.internet import defer
 from twisted.python import log
 
 from buildbot import interfaces, util
 from buildbot.changes.changes import Change
 from buildbot.process import buildstep
 from buildbot.process.properties import WithProperties
 from buildbot.status import builder
 from buildbot.steps import shell
 from buildbot.steps import source
 import sqlalchemy as sa
 
 from common import annotator
+from master import buildbucket
 from common import chromium_utils
 import config
 
 
 def change_to_revision(c):
   """Handle revision == None or any invalid value."""
   try:
     return int(str(c.revision).split('@')[-1])
   except (ValueError, TypeError):
     return 0
@@ skipping 595 matching lines @@
     self.annotate_status = builder.SUCCESS
     self.halt_on_failure = False
     self.honor_zero_return_code = False
     self.cursor = None
 
     self.show_perf = show_perf
     self.perf_id = perf_id
     self.perf_report_url_suffix = perf_report_url_suffix
     self.target = target
     self.active_master = active_master
+    self.bb_triggering_service = None
 
   def initialSection(self):
     """Initializes the annotator's sections.
 
     Annotator uses a list of dictionaries which hold information stuch as status
     and logs for each added step. This method populates the section list with an
     entry referencing the original buildbot step."""
     if self.sections:
       return
     # Add a log section for output before the first section heading.
@@ skipping 128 matching lines @@
     section['closed'] = True
     if section['step'].isFinished():
       return
 
     # Everything was fine on the slave side,
     # so the final result depends on async operations.
     async_ops = section['async_ops']
     if async_ops:
       op_list = '\n'.join('* %s' % o['description']
                           for o in async_ops)
-      msg = 'Will wait till async operations complete:\n%s' % (op_list,)
+      msg = 'Will wait till async operations complete:\n%s\n\n' % (op_list,)
       section['log'].addStdout(msg)
 
     d = defer.DeferredList([o['deferred'] for o in async_ops])
     def finish(results):
       try:
         reasons = []
         status = section['status']
         for succeeded, defer_result in results:
           if succeeded:
             # callback was called.
@@ skipping 369 matching lines @@
     files = change.get('files')
     if files:
       change['files'] = sorted(files)
 
     return change
 
   def STEP_TRIGGER(self, spec):
     # Support: @@@STEP_TRIGGER <json spec>@@@ (trigger build(s)).
     try:
       spec = json.loads(spec)
+      bucket = spec.get('bucket')
       builder_names = spec.get('builderNames')
-
+      properties = spec.get('properties') or {}
       changes = spec.get('changes')
       if changes:
         assert isinstance(changes, list)
         changes = map(self.normalizeChangeSpec, changes)
 
       if not builder_names:
         raise ValueError('builderNames is not specified: %r' % (spec,))
 
-      # Start builds.
-      d = self.triggerBuilds(builder_names, spec.get('properties') or {},
-                             changes)
+      build = self.command.build
+      build_is_from_buildbucket = bool(
+          build.getProperties().getProperty(buildbucket.common.INFO_PROPERTY))
+      trigger_via_buildbucket = bucket or build_is_from_buildbucket
+
+      if trigger_via_buildbucket:
+        d = self.triggerBuildsViaBuildBucket(
+            bucket, builder_names, properties, changes)
+      else:
+        d = self.triggerBuildsAsBuildsets(builder_names, properties, changes)

[ghost stip (do not use), 2015/03/04 23:07:31]

can you rename this to triggerBuildsLocally() instead?

[nodir, 2015/03/04 23:52:10]

Done

       # addAsyncOpToCursor expects a deferred to return a build result. If a
       # buildset is added, then it is a success. This lambda function returns a
       # tuple, which is received by addAsyncOpToCursor.
       d.addCallback(lambda _: (builder.SUCCESS, None))
       description = 'Triggering build(s) on %s' % (', '.join(builder_names),)
       self.addAsyncOpToCursor(d, description)
     except Exception as ex:
       self.finishStep(self.cursor, builder.FAILURE, ex)
 
   @staticmethod
@@ skipping 12 matching lines @@
     """Inserts a new SourceStamp.
 
     For each change in changes_spec, finds an existing or creates a new Change
     object. Then creates a SourceStamp with these changes.
 
     Args:
       master: an instance of buildbot.master.BuildMaster.
       changes_spec (list of dict): a list of change dicts, where each contains
         keyword arguments for
         buildbot.db.changes.ChangesConnectorComponent.addChange() function,
-        except when_timestamp is int instead of datetime. The first change
-        is used to populate source stamp properties.
+        except when_timestamp is int (seconds since Unix Epoch) instead of
+        datetime. The first change is used to populate source stamp properties.
     """
     def find_changes_by_revision(revision):
       """Searches for Changes in db by |revision| and returns change ids."""
       def find(conn):
         table = master.db.model.changes
         q = sa.select([table.c.changeid]).where(table.c.revision == revision)
         return [c.changeid for c in conn.execute(q)]
       return master.db.pool.do(find)
 
     @defer.inlineCallbacks
@@ skipping 50 matching lines @@
     ssid = yield master.db.sourcestamps.addSourceStamp(
         branch=main_change.branch,
         revision=main_change.revision,
         repository=main_change.repository,
         project=main_change.project,
         changeids=[c.number for c in changes],
     )
     defer.returnValue(ssid)
 
   @defer.inlineCallbacks
-  def triggerBuilds(self, builder_names, properties, changes=None):
+  def triggerBuildsViaBuildBucket(
+      self, bucket_name, builder_names, properties, changes_spec=None):
+    """Schedules builds on buildbucket."""
+    if self.active_master is None:
+      raise buildbucket.Error(
+          'In order to trigger builds through buildbucket, '
+          'ActiveMaster must be passed to AnnotatorFactory')
+    build = self.command.build
+    section = self.cursor
+    if not self.bb_triggering_service:

[ghost stip (do not use), 2015/03/04 23:07:30]

ok, so the service is only instantiated when something is actually triggered

[nodir, 2015/03/04 23:52:10]

Yes, when something is actually triggered through buildbucket

+      self.bb_triggering_service = yield (
+          buildbucket.trigger.get_triggering_service(self.active_master))
+    changes = map(
+        buildbucket.trigger.change_from_change_spec, changes_spec or [])
+    for builder_name in builder_names:
+      result = yield self.bb_triggering_service.trigger(
+          build, bucket_name, builder_name, properties, changes)
+      response = result['response']
+      section['log'].addStdout(
+          'BuildBucket.put API response: %s\n' % json.dumps(response, indent=4))
+      build_url = result.get('build_url')
+      if build_url:
+        section['log'].addStdout('See %s\n\n' % build_url)
+
+
+  @defer.inlineCallbacks
+  def triggerBuildsAsBuildsets(

[ghost stip (do not use), 2015/03/04 23:07:30]

triggerBuildsLocally

[nodir, 2015/03/04 23:52:10]

Done

+      self, builder_names, properties, changes_spec=None):
     """Creates a new buildset."""
     build = self.command.build
     master = build.builder.botmaster.parent
     current_properties = build.getProperties()
 
-    if changes:
+    if changes_spec:
       # Changes have been specified explicitly.
-      ssid = yield self.insertSourceStamp(master, changes)
+      ssid = yield self.insertSourceStamp(master, changes_spec)
     else:
       # Use the same source stamp.
       source_stamp = build.getSourceStamp()
       if source_stamp.revision is None:
         revision = current_properties.getProperty('got_revision')
         if revision:
           # The source stamp is relative, but we have the "got_revision", so
           # make it absolute.
           source_stamp = source_stamp.getAbsoluteSourceStamp(revision)
       else:
@@ skipping 151 matching lines @@
       and starts to wait for remaining steps to finish. Returns command_result
       as Deferred."""
       self.scriptComplete(command_result)
       steps_d = self.script_observer.waitForSteps()
       # Ignore the waitForSteps' result and return the original result,
       # so the caller of runCommand receives command_result.
       steps_d.addCallback(lambda *_: command_result)
       return steps_d
     d.addCallback(onCommandFinished)
     return d