Initial dart FYI buildbot setup.

masters/master.client.dart.fyi/master.cfg

+# -*- python -*-
+# ex: set syntax=python:
+
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# It has one job: define a dictionary named BuildmasterConfig. This
+# dictionary has a variety of keys to control different aspects of the
+# buildmaster. They are documented in docs/config.xhtml .
+
+from buildbot import locks
+from buildbot.changes import svnpoller
+from buildbot.scheduler import Dependent
+from buildbot.scheduler import Scheduler
+from buildbot.scheduler import Periodic
+
+from common import chromium_utils
+from master import build_utils
+from master import chromium_step
+from master import master_utils
+from master import slaves_list
+from master.factory import chromium_factory
+from master.factory import gclient_factory
+from master.factory import dart_factory
+from buildbot.process.buildstep import RemoteShellCommand
+
+import config
+ActiveMaster = config.Master.DartFYI
+
+# Hack to increase timeout for steps, dart2js debug checked mode takes more than
+# 8 hours.
+RemoteShellCommand.__init__.im_func.func_defaults = (None,

[nsylvain, 2012/04/12 00:38:17]

Someone modified AddTestStep in the last week or 2 to add a parameter to control
that.  In theory you would not have to hack it anymore.

+                                                     1,
+                                                     1,
+                                                     1200,
+                                                     48*60*60, {},
+                                                     'slave-config',
+                                                     True)
+
+MASTER_HOST = ActiveMaster.master_host
+WEB_STATUS = True
+MAIL_NOTIFIER = False
+TREE_GATE_KEEPER = False
+GOOD_REVISIONS = False
+MASTER_PORT = ActiveMaster.master_port
+
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)
+
+# 'slavePortnum' defines the TCP port to listen on. This must match the value
+# configured into the buildslaves (with their --master option)
+c['slavePortnum'] = ActiveMaster.slave_port
+
+# Setup a per slave lock to prevent more than one thing running at once on
+# a single slave.
+slave_lock = locks.SlaveLock('overload_lock', maxCount=1)
+
+
+# Slave allocation
+# build-base-name, category, platform, builder, tester
+variants = []
+
+
+slaves = slaves_list.SlavesList('slaves.cfg', 'Dart')
+
+
+####### CHANGESOURCES
+
+def DartTreeFileSplitter(path):
+  pieces = path.split('/')
+  if pieces[0] == 'trunk':
+    return ('trunk', '/'.join(pieces[1:]))
+  elif pieces[0] == 'branches':
+    return ('/'.join(pieces[0:2]),
+            '/'.join(pieces[2:]))
+  else:
+    return None
+
+dart_revision_url = "http://code.google.com/p/dart/source/detail?r=%s"
+
+# Polls config.Master.dart_url for changes
+poller = svnpoller.SVNPoller(svnurl=config.Master.dart_url,
+                             split_file=DartTreeFileSplitter,
+                             pollinterval=10,
+                             revlinktmpl=dart_revision_url)
+
+c['change_source'] = [poller]
+
+####### SCHEDULERS
+
+## configure the Schedulers
+
+builder_names = []
+for v in variants:
+  builder_names.append(v['name'])
+
+s = Scheduler(
+  name='main',
+  branch="branches/bleeding_edge",
+  treeStableTimer=0,
+  builderNames=builder_names
+)
+
+c['schedulers'] = [s]
+
+
+####### BUILDERS
+
+# buildbot/process/factory.py provides several BuildFactory classes you can
+# start with, which implement build processes for common targets (GNU
+# autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
+# base class, and is configured with a series of BuildSteps. When the build
+# is run, the appropriate buildslave is told to execute each Step in turn.
+
+# the first BuildStep is typically responsible for obtaining a copy of the
+# sources. There are source-obtaining Steps in buildbot/process/step.py for
+# CVS, SVN, and others.
+
+builders = []
+
+# ----------------------------------------------------------------------------
+# FACTORIES
+
+factory_base = {
+    'vm-mac': dart_factory.DartFactory('dart', 'vm-mac'),
+    'vm-linux': dart_factory.DartFactory('dart', 'vm-linux'),
+    'vm-win32': dart_factory.DartFactory('dart', 'vm-win32'),
+    'dartc-linux': dart_factory.DartFactory('dart', 'dartc-linux'),
+    'dart_client': dart_factory.DartFactory('dart', 'dart_client'),
+    'dart-editor': dart_factory.DartFactory('dart', 'dart-editor'),
+    'frog': dart_factory.DartFactory('dart', 'frog'),
+    'frogsh': dart_factory.DartFactory('dart', 'frogsh'),
+}
+
+for v in variants:
+  platform = v['platform']
+  base = factory_base[platform]
+  if platform in ['dart_client', 'dart-editor']:
+    v['factory_builder'] = base.DartAnnotatedFactory(
+        python_script='dart/client/tools/buildbot_annotated_steps.py',
+    )
+  else:
+    v['factory_builder'] = base.DartFactory(
+        slave_type='BuilderTester',
+        clobber=False,
+        options={
+            'mode': v['mode'],
+            'arch': v['arch'],
+            'name': v['name'],
+        },
+    )
+
+
+factories = []
+
+
+primary_builders = []
+for f in factories:
+  primary_builders.append(f[0])
+
+s_dartium = Scheduler(
+    name='dartium',
+    branch='branches/bleeding_edge',
+    treeStableTimer=0,
+    builderNames=primary_builders)
+c['schedulers'].append(s_dartium)
+
+# ----------------------------------------------------------------------------
+# BUILDER DEFINITIONS
+
+# The 'builders' list defines the Builders. Each one is configured with a
+# dictionary, using the following keys:
+#  name (required): the name used to describe this builder
+#  slavename (required): which slave to use, must appear in c['slaves']
+#  builddir (required): which subdirectory to run the builder in
+#  factory (required): a BuildFactory to define how the build is run
+#  periodicBuildTime (optional): if set, force a build every N seconds
+#  category (optional): it is not used in the normal 'buildbot' meaning. It is
+#                       used by gatekeeper to determine which steps it should
+#                       look for to close the tree.
+#
+
+c['builders'] = []
+for v in variants:
+  c['builders'].append({
+     'name': v['name'],
+     'builddir': v['name'],
+     'factory': v['factory_builder'],
+     'slavenames': slaves.GetSlavesName(builder=v['name']),
+     'category': v['category'],
+     'locks': [slave_lock],
+  })
+
+
+for f in factories:
+  c['builders'].append({
+      'name': f[0],
+      'slavenames': slaves.GetSlavesName(builder=f[0]),
+      'builddir': f[0],
+      'factory': f[2],
+      'category': '%s' % f[1],
+      # Don't enable auto_reboot for people testing locally.
+      'auto_reboot': ActiveMaster.is_production_host,
+  })
+
+####### BUILDSLAVES
+
+# The 'slaves' list defines the set of allowable buildslaves. List all the
+# slaves registered to a builder. Remove dupes.
+c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
+                                           config.Master.GetBotPassword())
+
+# Make sure everything works together.
+master_utils.VerifySetup(c, slaves)
+
+
+####### STATUS TARGETS
+
+# 'status' is a list of Status Targets. The results of each build will be
+# pushed to these targets. buildbot/status/*.py has a variety to choose from,
+# including web pages, email senders, and IRC bots.
+
+c['status'] = []
+
+if WEB_STATUS:
+  c['status'].append(
+      master_utils.CreateWebStatus(MASTER_PORT, allowForce=True,
+                                   public_html='./public_html',
+                                   templates=['./templates']))
+  c['status'].append(
+      master_utils.CreateWebStatus(ActiveMaster.master_port_alt,
+                                   allowForce=False))
+
+if MAIL_NOTIFIER:
+  from buildbot.status.mail import MailNotifier
+  c['status'].append(MailNotifier(fromaddr=ActiveMaster.from_address,
+                                  mode='problem',
+                                  sendToInterestedUsers=True,
+                                  extraRecipients=['ricow@google.com'],
+                                  lookup=master_utils.FilterDomain()))
+
+if TREE_GATE_KEEPER:
+  from master import gatekeeper
+  # This is the list of the builder categories and the corresponding critical
+  # steps. If one critical step fails, gatekeeper will close the tree
+  # automatically.
+  categories_steps = {
+      '': ['update scripts', 'update', 'clobber', 'clobber_packages'],
+      'closer': ['update scripts', 'update', 'compile', 'unit_tests'],
+      'info': [],
+      'inprogress': [],
+      'plugin': ['clobber', 'compile',
+                 'small_tests', 'medium_tests', 'large_tests'],
+      'qa': ['unit_tests'],
+  }
+  exclusions = {}
+  forgiving_steps = ['update scripts', 'update', 'svnkill', 'taskkill',
+                     'archived build']
+  c['status'].append(gatekeeper.GateKeeper(
+      fromaddr=ActiveMaster.from_address,
+      categories_steps=categories_steps,
+      exclusions=exclusions,
+      relayhost=config.Master.smtp,
+      subject='buildbot %(result)s in %(projectName)s on %(builder)s, '
+              'revision %(revision)s',
+      extraRecipients=ActiveMaster.tree_closing_notification_recipients,
+      lookup='google.com',
+      forgiving_steps=forgiving_steps))
+
+if GOOD_REVISIONS:
+  from master import goodrevisions
+  # This is the list of builders with their respective list of critical steps
+  # that all need to succeed to mark a revision as successful. A single failure
+  # in any of the steps of any of the builders will mark the revision as failed.
+  good_revision_steps = {
+      '': ['update', 'compile'],
+  }
+  c['status'].append(goodrevisions.GoodRevisions(
+      good_revision_steps=good_revision_steps,
+      store_revisions_url=ActiveMaster.store_revisions_url))
+
+
+# Keep last build logs, the default is too low.
+c['buildHorizon'] = 1000
+c['logHorizon'] = 500
+# Must be at least 2x the number of slaves.
+c['eventHorizon'] = 200
+# Must be at least 1x the number of builds listed in console.
+c['buildCacheSize'] = 60
+
+
+####### DEBUGGING OPTIONS
+
+# if you set 'debugPassword', then you can connect to the buildmaster with
+# the diagnostic tool in contrib/debugclient.py . From this tool, you can
+# manually force builds and inject changes, which may be useful for testing
+# your buildmaster without actually commiting changes to your repository (or
+# before you have a functioning 'sources' set up). The debug tool uses the
+# same port number as the slaves do: 'slavePortnum'.
+
+#c['debugPassword'] = 'debugpassword'
+
+# if you set 'manhole', you can ssh into the buildmaster and get an
+# interactive python shell, which may be useful for debugging buildbot
+# internals. It is probably only useful for buildbot developers. You can also
+# use an authorized_keys file, or plain telnet.
+#from buildbot import manhole
+#c['manhole'] = manhole.PasswordManhole('tcp:9999:interface=127.0.0.1',
+#                                       'admin', 'password')
+
+
+####### PROJECT IDENTITY
+
+# the 'projectName' string will be used to describe the project that this
+# buildbot is working on. For example, it is used as the title of the
+# waterfall HTML page. The 'projectURL' string will be used to provide a link
+# from buildbot HTML pages to your project's home page.
+
+c['projectName'] = ActiveMaster.project_name
+c['projectURL'] = config.Master.project_url
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server (usually the html.Waterfall page) is visible. This
+# typically uses the port number set in the Waterfall 'status' entry, but
+# with an externally-visible host name which the buildbot cannot figure out
+# without some help.
+
+c['buildbotURL'] = 'http://build.chromium.org/p/client.dart/'