Add ability to specify email subject for gatekeeper_ng.

scripts/slave/gatekeeper_ng.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Closes tree if configured masters have failed tree-closing steps.
 
 Given a list of masters, gatekeeper_ng will get a list of the latest builds from
 the specified masters. It then checks if any tree-closing steps have failed, and
 if so closes the tree and emails appropriate parties. Configuration for which
@@ skipping 135 matching lines @@
   failed_builds = []
   for build_json, master_url in master_builds:
     gatekeeper_sections = gatekeeper_config.get(master_url, [])
     for gatekeeper_section in gatekeeper_sections:
       gatekeeper = gatekeeper_section.get(build_json['builderName'], {})
       steps = build_json['steps']
       forgiving = set(gatekeeper.get('forgiving_steps', []))
       closing_steps = set(gatekeeper.get('closing_steps', [])) | forgiving
       tree_notify = set(gatekeeper.get('tree_notify', []))
       sheriff_classes = set(gatekeeper.get('sheriff_classes', []))
+      subject_template = gatekeeper['subject_template']
       finished = [s for s in steps if s.get('isFinished')]
       close_tree = gatekeeper.get('close_tree', True)
       respect_build_status = gatekeeper.get('respect_build_status', False)
 
       successful_steps = set(s['name'] for s in finished
                              if (s.get('results', [FAILURE])[0] == SUCCESS or
                                  s.get('results', [FAILURE])[0] == WARNINGS))
 
       unsatisfied_steps = closing_steps - successful_steps
 
@@ skipping 28 matching lines @@
         build_db[master_url][build_json['builderName']] = max(
             build_json['number'],
             build_db[master_url][build_json['builderName']])
 
         failed_builds.append({'base_url': buildbot_url,
                               'build': build_json,
                               'close_tree': close_tree,
                               'forgiving_steps': forgiving,
                               'project_name': project_name,
                               'sheriff_classes': sheriff_classes,
+                              'subject_template': subject_template,
                               'tree_notify': tree_notify,
                               'unsatisfied': unsatisfied_steps,
                              })
 
   return failed_builds
 
 
 def allowed_keys(test_dict, *keys):
   keys = keys + ('comment',)
   assert all(k in keys for k in test_dict), (
@@ skipping 22 matching lines @@
   is considered blank (empty set), and inheritance is always constructive (you
   can't remove a property by inheriting or overwriting it). Builders can inherit
   categories from their master.
 
   A master consists of zero or more sections, which specify which builders are
   watched by the section and what action should be taken. A section can specify
   tree_closing to be false, which causes the section to only send out emails
   instead of closing the tree. A section or builder can also specify to respect
   a build's failure status with respect_build_status.
 
+  The 'subject_template' key is the template used for the email subjects. Its
+  formatting arguments are found at https://chromium.googlesource.com/chromium/
+    tools/chromium-build/+/master/gatekeeper_mailer.py, but the list is
+  reproduced here:
+
+    %(result)s: 'warning' or 'failure'
+    %(projectName): 'Chromium', 'Chromium Perf', etc.
+    %(builder): the builder name
+    %(reason): reason for launching the build
+    %(revision): build revision
+    %(buildnumber): buildnumber
+
   The 'comment' key can be put anywhere and is ignored by the parser.
 
   # Python, not JSON.
   {
     'masters': {
       'http://build.chromium.org/p/chromium.win': [
         {
           'sheriff_classes': ['sheriff_win'],
           'tree_notify': ['a_watcher@chromium.org'],
           'categories': ['win_extra'],
@@ skipping 12 matching lines @@
     'categories': {
       'win_tests': {
         'comment': 'this is for all windows testers',
         'closing_steps': ['startup_test'],
         'forgiving_steps': ['boot_windows'],
         'tree_notify': ['win_watchers@chromium.org'],
         'sheriff_classes': ['sheriff_win_test']
       },
       'win_extra': {
         'closing_steps': ['extra_win_step']
+        'subject_template': 'windows heads up on %(builder)',
       }
     }
   }
 
   In this case, XP Tests (1) would be flattened down to:
     closing_steps: ['startup_test', 'win_tests']
     forgiving_steps: ['archive', 'boot_windows']
     tree_notify: ['xp_watchers@chromium.org', 'win_watchers@chromium.org',
                   'a_watcher@chromium.org']
     sheriff_classes: ['sheriff_win', 'sheriff_win_test', 'sheriff_xp']
 
   Again, fields are optional and treated as empty lists/sets if not present.
   """
 
-  master_keys = ['sheriff_classes', 'tree_notify']
+  # Keys which are allowed in a master or builder section.
+  master_keys = ['sheriff_classes', 'tree_notify', 'subject_template']
   builder_keys = ['forgiving_steps', 'closing_steps', 'tree_notify',
-                  'sheriff_classes']
+                  'sheriff_classes', 'subject_template']
+
+
+  # Keys which have defaults besides None or set([]).
+  defaults = {
+      'subject_template': ('buildbot %(result)s in %(projectName)s on '
+                           '%(builder)s, revision %(revision)s'),
+  }
+
+  # These keys are strings instead of sets. Strings can't be merged,
+  # so more specific (master -> category -> builder) strings clobber
+  # more generic ones.
+  strings = ['subject_template']
 
   with open(filename) as f:
     raw_gatekeeper_config = json.load(f)
 
   allowed_keys(raw_gatekeeper_config, 'categories', 'masters')
 
   categories = raw_gatekeeper_config.get('categories', {})
   masters = raw_gatekeeper_config.get('masters', {})
 
   for category in categories.values():
     allowed_keys(category, *builder_keys)
 
   gatekeeper_config = {}
   for master_url, master_sections in masters.iteritems():
     for master_section in master_sections:
       gatekeeper_config.setdefault(master_url, []).append({})
       allowed_keys(master_section, 'builders', 'categories', 'close_tree',
                    'respect_build_status', *master_keys)
 
       builders = master_section.get('builders', {})
       for buildername, builder in builders.iteritems():
         allowed_keys(builder, 'categories', *builder_keys)
-        for item in builder.values():
-          assert isinstance(item, list)
-          assert all(isinstance(elem, basestring) for elem in item)
+        for key, item in builder.iteritems():
+          if key in strings:
+            assert isinstance(item, basestring)
+          else:
+            assert isinstance(item, list)
+            assert all(isinstance(elem, basestring) for elem in item)
 
         gatekeeper_config[master_url][-1].setdefault(buildername, {})
         gatekeeper_builder = gatekeeper_config[master_url][-1][buildername]
 
-        # Populate with empty defaults.
+        # Populate with specified defaults.
         for k in builder_keys:
-          gatekeeper_builder.setdefault(k, set())
+          if k in defaults:
+            gatekeeper_builder.setdefault(k, defaults[k])
+          elif k in strings:
+            gatekeeper_builder.setdefault(k, '')
+          else:
+            gatekeeper_builder.setdefault(k, set())
 
         # Inherit any values from the master.
         for k in master_keys:
-          gatekeeper_builder[k] |= set(master_section.get(k, []))
+          if k in strings:
+            if k in master_section:
+              gatekeeper_builder[k] = master_section[k]
+          else:
+            gatekeeper_builder[k] |= set(master_section.get(k, []))
 
         gatekeeper_builder['close_tree'] = master_section.get('close_tree',
                                                               True)
         gatekeeper_builder['respect_build_status'] = master_section.get(
             'respect_build_status', False)
 
         # Inherit any values from the categories.
         all_categories = (builder.get('categories', []) +
                           master_section.get( 'categories', []))
         for c in all_categories:
           for k in builder_keys:
-            gatekeeper_builder[k] |= set(categories[c].get(k, []))
+            if k in strings:
+              if k in categories[c]:
+                gatekeeper_builder[k] = categories[c][k]
+            else:
+              gatekeeper_builder[k] |= set(categories[c].get(k, []))
 
         # Add in any builder-specific values.
         for k in builder_keys:
-          gatekeeper_builder[k] |= set(builder.get(k, []))
+          if k in strings:
+            if k in builder:
+              gatekeeper_builder[k] = builder[k]
+          else:
+            gatekeeper_builder[k] |= set(builder.get(k, []))
 
   return gatekeeper_config

[iannucci, 2013/12/17 00:46:23]

This whole function is pretty funky. It may be worth thinking about how to
refactor this so that the bare python types (dict, str, etc.) get parsed into a
stronger more-typed hierarchy of composed classes.

[ghost stip (do not use), 2013/12/17 18:37:14]

Good point, might make it a bit easier to traverse and verify.

 
 
 def parse_sheriff_file(url):
   """Given a sheriff url, download and parse the appropirate sheriff list."""
   with closing(urllib2.urlopen(url)) as f:
     line = f.readline()
   usernames_matcher_ = re.compile(r'document.write\(\'([\w, ]+)\'\)')
   usernames_match = usernames_matcher_.match(line)
   sheriffs = set()
   if usernames_match:
@@ skipping 83 matching lines @@
     tree_notify = failed_build['tree_notify']
 
     if failed_build['unsatisfied'] <= failed_build['forgiving_steps']:
       blamelist = set()
     else:
       blamelist = set(failed_build['build']['blame'])
 
     sheriffs = get_sheriffs(failed_build['sheriff_classes'], sheriff_url)
     watchers = list(tree_notify | blamelist | sheriffs)
 
-
     build_data = {
         'build_url': build_url,
         'from_addr': fromaddr,
         'project_name': project_name,
+        'subject_template': failed_build['subject_template'],
         'steps': [],
         'unsatisfied': list(failed_build['unsatisfied']),
         'waterfall_url': waterfall_url,
     }
 
     for field in ['builderName', 'number', 'reason']:
       build_data[field] = failed_build['build'][field]
 
     build_data['result'] = failed_build['build'].get('results', 0)
     build_data['blamelist'] = failed_build['build']['blame']
@@ skipping 191 matching lines @@
                         options.disable_domain_filter)
 
   if not options.skip_build_db_update:
     save_build_db(build_db, options.build_db)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())