CrOS: Update public waterfall to auto-configure.

masters/master.chromiumos/master.cfg

Index: masters/master.chromiumos/master.cfg
diff --git a/masters/master.chromiumos/master.cfg b/masters/master.chromiumos/master.cfg
index 651b4a735c9e1c5f3b3b5bc9092c6415790528cb..f679240ba3830393e749c9a7d9265f58abebe05c 100644
--- a/masters/master.chromiumos/master.cfg
+++ b/masters/master.chromiumos/master.cfg
@@ -14,6 +14,8 @@
 # dictionary has a variety of keys to control different aspects of the
 # buildmaster. They are documented in docs/config.xhtml .
 
+import json
+import os
 import re
 
 from buildbot.changes.filter import ChangeFilter
@@ -30,21 +32,21 @@ from master.cros import builder_config
 from master.factory import chromeos_factory
 
 # These modules come from scripts/common, which must be in the PYTHONPATH.
+import chromiumos_board_config
 import config
 import master_site_config
+from master.cros import builder_config
+from common.cros_chromite import ChromiteTarget
 
 ActiveMaster = master_site_config.ChromiumOS
-
-TREE_GATE_KEEPER = ActiveMaster.is_production_host
-
-# Never enable auto_reboot for folks testing locally.
-DISABLE_AUTO_REBOOT = not ActiveMaster.is_production_host
+DRY_RUN = not ActiveMaster.is_production_host
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
-BUILDERS = []
+config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)

[ghost stip (do not use), 2015/04/10 22:47:04]

review note: this is from line 183

+
 
 # ----------------------------------------------------------------------------
 # BUILDER DEFINITIONS
@@ -60,126 +62,69 @@ BUILDERS = []
 #                       look for to close the tree.
 #
 
-# Cros helper functions to build builders and factories.
-
-def GetCBuildbotFactory(config, **kwargs):
-  """Returns cros buildbot factories."""
-  return chromeos_factory.CbuildbotFactory(
-      params=config, dry_run=not ActiveMaster.is_production_host,
-      buildroot='/b/cbuild/external_master', show_gclient_output=False,
-      **kwargs).get_factory()
-
-
-TAGS = {}
-
-class BuilderDefinition(object):
-  def __init__(self, name):
-    self.name = name
+# General source to add in cbuildbot types:
+def GenCBuild(bc, root_dir=None, auto_reboot=False, branch='master',
+              slave_root='/b', extra_categories=None):
+  """Generate a cbuild buildbot configuration
+
+      Create a buildbot builder configuration and return a builder
+      dictionary associated with it.
+
+    Arguments:
+      bc: (builder_config.BuilderConfig) The config.
+      root_dir: Root of the directory where all work will take place.
+      name: Name as displayed in the waterfall, if None generate automatically.
+      auto_reboot: bool whether or not the target should reboot between builds
+      branch: The branch to set the builder up for, defaults to 'master'
+      slave_root: The root dir where the buildbot slave is installed.
+    Returns:
+      A builder dictionary assocaited with a factory
+  """
+  categories = ['1release full']
+  if bc.closer:
+    categories.append('closer')
+  else:
+    categories.append('info')
 
-  def tag(self, *tags):
-    for tag in tags:
-      TAGS.setdefault(tag, []).append(self.name)
-    return self
+  build_dir_parts = [str(bc.config.name)]
+  if branch:
+    build_dir_parts.append(branch)
 
+  # All builders on the internal waterfall should use the same directory names
+  # as the trybot waterfall and any other internal waterfalls. The "_master"
+  # suffix is for historical reasons -- we use that suffix regardless of what
+  # branch we're building for.
+  if root_dir is None:
+    root_dir = 'external_master'
 
-def AddBuilderDefinition(display_name, cbb_name, closer=True, auto_reboot=False,
-                         superTime=None, collapse=False):
-  """Adds a builder definition given by the args.
+  cbuild_root = os.path.join(slave_root, 'cbuild', root_dir)
 
-  Args:
-    display_name: Name displayed on buildbot waterfall.
-    closer:  Do we close the tree based on this build's failure.
-    auto_reboot: Whether to reboot the bot after each run.
-  """
-  category = '1release full|info'
-  builder_definition = BuilderDefinition(display_name)
-  if closer:
-    category = '1release full|closer'
-    builder_definition.tag('closer')
-
-  # Discard suffixes such as " (chromium:12345)".
-  build_dir = re.sub(r'\s\(.*\)$', '', display_name)
-  build_dir = build_dir.replace(' ', '-')
-  if superTime:
-    factory = GetCBuildbotFactory(cbb_name, max_time=superTime)
-  else:
-    factory = GetCBuildbotFactory(cbb_name)
+  kwargs = {}
+  # Give the SDK builder more time.
+  if bc.timeout:
+    kwargs['max_time'] = bc.timeout
 
-  if DISABLE_AUTO_REBOOT:
-    auto_reboot = False
+  factory = chromeos_factory.CbuildbotFactory(

[ghost stip (do not use), 2015/04/10 22:47:03]

review note: line 67

+      params=str(bc.config.name), branch=branch, buildroot=cbuild_root,

[ghost stip (do not use), 2015/04/10 22:47:03]

review note: 'config' in getcbuildbotfactory (line 65) is now bc.config.name,
which must be converted to string for some reason

branch is a new parameter

buildroot used to be hardcoded to /b/cbuild/external_master, now is cbuild_root
which is slave_root/cbuild/root_dir    ->  root_dir is passed into GenCBuild

+      show_gclient_output=False, dry_run=DRY_RUN, **kwargs

[ghost stip (do not use), 2015/04/10 22:47:03]

review note: DRY_RUN is line 42, same as 'not activemaster' on line 68

+      ).get_factory()
 
   builder = {

[ghost stip (do not use), 2015/04/10 22:47:03]

review note: GetCBuildbotFactory was called by AddBuilderDefinition, these are
both now collapsed into GenCBuild

-    'name': display_name,
-    'builddir': build_dir,
-    'factory': factory,
-    'category': category,
-    'auto_reboot': auto_reboot,
+      'name': str(bc.builder_name),
+      'builddir': '-'.join(build_dir_parts),
+      'factory': factory,
+      'category': '|'.join(categories),
+      'auto_reboot': auto_reboot,
   }
-  if collapse:
+  if bc.collapse:
     builder['collapseRequests'] = builder_config.AlwaysCollapseFunc
-  BUILDERS.append(builder)
-  return builder_definition
-
-
-def _AddBoardTarget(board, target_type, **kw):

[ghost stip (do not use), 2015/04/10 22:47:03]

review note: all these have been replaced by
chromiumos_board_config.builder_configs

-  return AddBuilderDefinition('%s %s' % (board, target_type),
-                              '%s-%s' % (board, target_type), **kw)
-
-def AddPaladin(board, **kw):
-  kw['closer'] = False
-  return _AddBoardTarget(board, 'paladin', **kw).tag('paladin')
-
-def AddIncremental(board, **kw):
-  kw.setdefault('collapse', True)
-  return _AddBoardTarget(board, 'incremental', **kw).tag('default')
-
-def AddFull(board, **kw):
-  kw.setdefault('collapse', True)
-  return _AddBoardTarget(board, 'full', **kw).tag('default')
-
-def AddASAN(board, **kw):
-  kw.setdefault('collapse', True)
-  return AddBuilderDefinition(
-      '%s ASAN' % (board,),
-      '%s-asan' % (board,), **kw).tag('default')
-
-# Paladin Builders -- exception to closer rule below as they are very important
-# to watch.
-AddPaladin('x86-generic')
-AddPaladin('amd64-generic')
-AddPaladin('amd64-generic_freon')
-AddPaladin('gizmo')
-AddPaladin('mipsel-o32-generic')
-AddPaladin('arm-generic')
-AddPaladin('panther_embedded-minimal')
+  return builder
 
-AddIncremental('x86-generic').tag('x86')
-AddIncremental('amd64-generic').tag('x86')
-AddIncremental('daisy').tag('arm')
-
-# Full Builders
-AddFull('x86-generic').tag('x86')
-AddFull('amd64-generic').tag('x86')
-AddFull('daisy').tag('arm')
-AddFull('arm-generic').tag('arm')
-AddFull('mipsel-o32-generic')
-
-AddBuilderDefinition('chromiumos sdk', 'chromiumos-sdk',
-                     superTime=22*3600, collapse=True).tag('default')
-
-####### Non Closer build defs.
-
-# Miscellaneous builders.
-AddBuilderDefinition('refresh packages (chromium:412795)', 'refresh-packages',
-                     closer=False).tag('refresh')
-AddASAN('x86-generic')
-AddASAN('amd64-generic', closer=False)
-
-
-c['builders'] = BUILDERS
+# Associate the slaves to the builders.
+c['builders'] = []
 
-
-config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)
+for bc in chromiumos_board_config.builder_configs.itervalues():
+  c['builders'].append(GenCBuild(bc))
 
 
 ####### CHANGESOURCES
@@ -196,8 +141,23 @@ c['change_source'].append(CommentRespectingGitPoller(
 
 ####### SCHEDULERS
 
+def GetBuilders(func):
+  return [b for b in chromiumos_board_config.builder_configs.itervalues()
+          if func(b)]
+
+
+def GetBuilderNames(func):
+  return [str(builder_config.builder_name)
+          for builder_config in GetBuilders(func)]
+
+
+def GetBuilderNamesForCategory(category):
+  return GetBuilderNames(lambda b: b.config.category == category)
+
+
 ## configure the Schedulers
-# XXX: Changes to builderNames must also be made in:# - slaves.cfg
+# XXX: Changes to builderNames must also be made in:
+# - slaves.cfg
 # - templates/announce.html
 # - And down below in the builder definitions as well
 # - and you probably need to restart any changed slaves as well as the master
@@ -205,13 +165,13 @@ c['change_source'].append(CommentRespectingGitPoller(
 s_paladin = ChromeOSManifestSingleBranchScheduler(
   name='paladin',
   change_filter=FilterNewSpec(MANIFEST_VERSIONS_REPO, 'master-paladin'),
-  builderNames=TAGS['paladin'],
+  builderNames=GetBuilderNamesForCategory(ChromiteTarget.PALADIN),
   )
 
 s_refresh_packages = Periodic(
   name='refresh_pkgs_scheduler',
   periodicBuildTimer=24 * 60 * 60, # 1 day
-  builderNames=TAGS['refresh'],
+  builderNames=GetBuilderNamesForCategory(ChromiteTarget.REFRESH_PACKAGES),
   )
 
 # Default scheduler triggers when we see changes.
@@ -219,7 +179,9 @@ repository_fn = lambda x: x != MANIFEST_VERSIONS_REPO
 s_chromeos_default = ChromeOSManifestAnyBranchScheduler(
   name='chromeos',
   change_filter=ChangeFilter(repository_fn=repository_fn, branch='master'),
-  builderNames=TAGS['default'],
+  builderNames=(
+      GetBuilderNamesForCategory(ChromiteTarget.INCREMENTAL) +
+      GetBuilderNamesForCategory(ChromiteTarget.FULL))
   )
 
 c['schedulers'] = [
@@ -234,8 +196,8 @@ c['schedulers'] = [
 
 # First, load the list from slaves.cfg.
 slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumOS')
-
-# Associate the slaves to the builders.
+if not slaves.GetSlaves():
+  raise ValueError("Failed to load slaves.")
 for builder in c['builders']:
   builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
 
@@ -257,23 +219,27 @@ master_utils.AutoSetupMaster(c, ActiveMaster,
     order_console_by_time=True)
 
 ####### BUILDER LIST OUTPUT
-def write_js_reference(builder_groups):
-  """Generate a js file for the waterfall to include."""
 
+def write_js_json(varname, d):
+  """Generate a js file for the waterfall to include.
+
+  We do this by creating a Javascript fragment defining the variable, 'varname',
+  to be the result of parsing emitted JSON.
+  """
+  json_dump = json.dumps(d, indent=2, sort_keys=True)
+  data = 'var %s = %s;' % (varname, json_dump)
   with open('public_html/auto-builder.js', 'w') as f:
-    for name, builders in builder_groups:
-      f.write('var %s = [\n' % name)
-      f.write(',\n'.join(['   "builder=%s"' % b for b in builders]))
-      f.write('  ].join("&");\n')
+    f.write(data)
 
 # This gets called by the shim when we need to write the JS file(s).
 def WriteHTMLFragments():
-  write_js_reference([
-      ('closer', TAGS['closer']),
-      ('x86', TAGS['x86']),
-      ('arm', TAGS['arm']),
-      ('commit_queue', TAGS['paladin']),
-      ])
+  write_js_json('buildercfg', {
+    'closers': GetBuilderNames(lambda b: b.closer),
+    'paladin': GetBuilderNamesForCategory(ChromiteTarget.PALADIN),
+    'incremental': GetBuilderNamesForCategory(ChromiteTarget.INCREMENTAL),
+    'asan': GetBuilderNamesForCategory(ChromiteTarget.ASAN),
+    'full': GetBuilderNamesForCategory(ChromiteTarget.FULL),
+  })
 
 ####### TROOPER NAGGING
 if ActiveMaster.is_production_host:
@@ -297,4 +263,17 @@ if ActiveMaster.is_production_host:
       use_getname=True))
 
 if not ActiveMaster.is_production_host:
+  # Save our slave pool state. This is populated when our 'slaves' variable

[ghost stip (do not use), 2015/04/10 22:47:03]

reviewer note: this is all for when is_production_host is false

+  # gets generated.
+  chromiumos_board_config.slave_allocator.SaveState(list_unallocated=True)
+  slave_map = chromiumos_board_config.slave_allocator.GetSlaveMap()
+  if slave_map.unallocated:
+    log.msg("The following slaves were not allocated: %s" % (
+        sorted(slave_map.unallocated),))
+
+  # Dump the current configuration.
   master_utils.DumpSetup(c)
+
+  # Disable 'auto_reboot' on slaves for local testing.

[ghost stip (do not use), 2015/04/10 22:47:03]

weird, I was under the impression  that the slave itself would detect
non-production and not reboot. guess not

[dnj, 2015/04/12 15:20:38]

I'm pretty sure it does, as I've never had a system reboot when running a slave.
Maybe they didn't originally? All of these waterfalls could use a cleanup.

+  for builder in c['builders']:
+    builder['auto_reboot'] = False