client.skia: separate builder and buildslave configs

masters/master.client.skia/master.cfg

 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 # vim: set ft=python:
 
+import collections
+
 from buildbot.changes.gitpoller import GitPoller
-
+from buildbot.scheduler import Scheduler
+from buildbot.scheduler import Triggerable
+from common import chromium_utils
 from master import build_utils
 from master import master_config
 from master import master_utils
-from master import recipe_master_helper
 from master import slaves_list
 from master.factory import annotator_factory
 
 import config
 import master_site_config
 
 
+def AddSchedulersAndTriggers(buildmaster_config,
+                             builders,
+                             scheduler_name,
+                             branch=None):
+  """Clone of recipe_master_helper.AddSchedulersAndTriggers which separates
+  builders and buildslaves more cleanly.
+  """
+  c = buildmaster_config
+  polling_builders = []
+  parent_builders = set()
+  # Maps the parent builder to a set of the names of the builders it triggers.
+  trigger_map = collections.defaultdict(list)
+  # Maps the name of the parent builder to the (synthesized) name of its
+  # trigger, wrapped in a list.
+  trigger_name_map = {}
+  next_group_id = 0
+  for builder in builders:
+    builder_name = builder['name']
+    parent_builder = builder.get('triggered_by')
+    if parent_builder is not None:
+      parent_builders.add(parent_builder)
+      trigger_map[parent_builder].append(builder_name)
+      if parent_builder not in trigger_name_map:
+        trigger_name_map[parent_builder] = 'trigger_group_%d' % next_group_id
+        next_group_id += 1
+    else:
+      polling_builders.append(builder_name)
+  # Verify that all parent builders exist.
+  nonexistent_parents = parent_builders - set(polling_builders)
+  if nonexistent_parents:
+    raise Exception('Could not find parent builders: %s' %
+                    ', '.join(nonexistent_parents))
+  s = Scheduler(name=scheduler_name,
+                branch=branch,
+                treeStableTimer=60,
+                builderNames=polling_builders)
+  c['schedulers'] = [s]
+  for name, builders in trigger_map.iteritems():
+    c['schedulers'].append(Triggerable(name=trigger_name_map[name],
+                                       builderNames=builders))
+  return trigger_name_map
+
+
+def AddRecipeBasedBuilders(buildmaster_config,
+                           builders,
+                           annotator,
+                           trigger_name_map,
+                           default_auto_reboot=False):
+  """Clone of recipe_master_helper.AddRecipeBasedBuilders which separates
+  builders and buildslaves more cleanly.
+  """
+  builders_list = []
+  for builder in builders:
+    if 'recipe' in builder:
+      name = builder['name']
+      build_config = 'Release' if 'Release' in name else 'Debug'
+      factory_properties = {
+        'test_results_server': 'test-results.appspot.com',
+        'generate_gtest_json': True,
+        'build_config': build_config,
+      }
+      if 'perf_id' in builder:
+        factory_properties['show_perf_results'] = True
+        factory_properties['perf_id'] = builder['perf_id']
+      builder_dict = {
+        'name': name,
+        'factory': annotator.BaseFactory(
+          builder['recipe'],
+          factory_properties,
+          [trigger_name_map[name]] if name in trigger_name_map else None),
+        'gatekeeper': builder.get('gatekeeper_categories', ''),
+      }
+      builder_dict['auto_reboot'] = builder.get('auto_reboot', False)
+      builders_list.append(builder_dict)
+  buildmaster_config['builders'] = builders_list
+
+
 ActiveMaster = master_site_config.Skia
 
 c = BuildmasterConfig = {}
 
 config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)
 
 ####### CHANGESOURCES
 
 # Polls config.Master.trunk_url for changes
 poller = GitPoller(
     repourl=ActiveMaster.repo_url,
     branch='master',
     pollinterval=10,
     revlinktmpl='https://skia.googlesource.com/skia/+/%s')
 
-
 c['change_source'] = [poller]
 
-
 ####### SLAVES
 
 # Load the slave list. We need some information from it in order to
 # produce the builders.
-slaves = slaves_list.SlavesList('slaves.cfg', 'Skia')
+slaves = slaves_list.SlavesList('slaves.cfg', ActiveMaster.project_name)
+
+####### BUILDERS
+
+# Load the builders list.
+builders = chromium_utils.ParsePythonCfg('builders.cfg')['builders']
 
 ####### SCHEDULERS
 
-## configure the Schedulers
+# Configure the Schedulers.
+trigger_name_map = AddSchedulersAndTriggers(buildmaster_config=c,
+                                            builders=builders,
+                                            scheduler_name='skia',
+                                            branch='master')
 
-# Main scheduler for all changes in trunk.
-
-trigger_name_map = recipe_master_helper.AddSchedulersAndTriggers(
-    buildmaster_config=c, slave_list=slaves, scheduler_name='skia',
-    branch='master')
-
-####### BUILDERS
-
-builders = []
-
-# ----------------------------------------------------------------------------
-# FACTORIES
+####### FACTORIES
 
 m_annotator = annotator_factory.AnnotatorFactory()
-
-recipe_master_helper.AddRecipeBasedBuilders(
-    c, slaves, m_annotator, trigger_name_map)
+AddRecipeBasedBuilders(c, builders, m_annotator, trigger_name_map)
 
 # Associate the slaves to the manual builders. The configuration is in
 # slaves.cfg.
 for builder in c['builders']:
   builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
 
 ####### BUILDSLAVES
 
 # The 'slaves' list defines the set of allowable buildslaves. List all the
 # slaves registered to a builder. Remove dupes.
 c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
                                            config.Master.GetBotPassword())
-
 master_utils.VerifySetup(c, slaves)
 
 ####### STATUS TARGETS
 
 c['buildbotURL'] = ActiveMaster.buildbot_url
 
 # Adds common status and tools to this master.
 master_utils.AutoSetupMaster(c, ActiveMaster,
     public_html='../master.chromium/public_html',
     templates=['../master.chromium/templates'],
     enable_http_status_push=ActiveMaster.is_production_host,
     order_console_by_time=True)