Reimplemented tree_for_master, which now uses gatekeeper config files to map master URLs to the tre…

infra/tools/builder_alerts/__main__.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import argparse
 import datetime
 import json
 import logging
 import sys
@@ skipping 15 matching lines @@
 build_scripts_dir = os.path.join(top_dir, 'build', 'scripts')
 sys.path.insert(0, build_scripts_dir)
 
 # Our sys.path hacks are too bursting with chest-hair for pylint's little brain.
 from slave import gatekeeper_ng_config  # pylint: disable=F0401
 
 
 CACHE_PATH = 'build_cache'
 
 
-def apply_gatekeeper_rules(alerts, gatekeeper):
-  filtered_alerts = []
-  for alert in alerts:
-    master_url = alert['master_url']
-    master_name = buildbot.master_name_from_url(master_url)
-    config = gatekeeper.get(master_url)
-    if not config:
-      # Unclear if this should be set or not?
-      # alert['would_close_tree'] = False
-      filtered_alerts.append(alert)
-      continue
-    excluded_builders = gatekeeper_extras.excluded_builders(config)
-    if alert['builder_name'] in excluded_builders:
-      continue
-    alert['would_close_tree'] = \
-      gatekeeper_extras.would_close_tree(config,
-        alert['builder_name'], alert['step_name'])
-    filtered_alerts.append(alert)
-    alert['tree_name'] = gatekeeper_extras.tree_for_master(master_name)
-  return filtered_alerts
-
-
-def fetch_master_urls(gatekeeper, args):
-  # Currently using gatekeeper.json, but could use:
-  # https://chrome-infra-stats.appspot.com/_ah/api#p/stats/v1/stats.masters.list
-  master_urls = gatekeeper.keys()
-  if args.master_filter:
-    master_urls = [url for url in master_urls if args.master_filter not in url]
-  return master_urls
-
-
 def main(args):
   parser = argparse.ArgumentParser(prog='run.py %s' % __package__)
   parser.add_argument('data_url', action='store', nargs='*')
   parser.add_argument('--use-cache', action='store_true')
   parser.add_argument('--master-filter', action='store')
   parser.add_argument('--builder-filter', action='store')
   # FIXME: Ideally we'd have adjustable logging instead of just DEBUG vs. CRIT.
   parser.add_argument("-v", "--verbose", action='store_true')
 
   gatekeeper_json = os.path.join(build_scripts_dir, 'slave', 'gatekeeper.json')
   parser.add_argument('--gatekeeper', action='store', default=gatekeeper_json)
   args = parser.parse_args(args)
 
   logging.basicConfig(level=logging.DEBUG if args.verbose else logging.CRITICAL)
 
   if not args.data_url:
     logging.warn("No /data url passed, will write to builder_alerts.json")
 
   if args.use_cache:
     requests_cache.install_cache('failure_stats')
   else:
     requests_cache.install_cache(backend='memory')
 
   # FIXME: gatekeeper_config should find gatekeeper.json for us.
   gatekeeper_path = os.path.abspath(args.gatekeeper)
-  print "Processsing gatekeeper json: %s" % (gatekeeper_path)
+  logging.debug("Processsing gatekeeper json: %s" % gatekeeper_path)
   gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
-  master_urls = fetch_master_urls(gatekeeper, args)
+
+  gatekeeper_trees_path = os.path.join(os.path.dirname(gatekeeper_path),
+                                       'gatekeeper_trees.json')
+  logging.debug('Processing gatekeeper trees json: %s' % gatekeeper_trees_path)

[tandrii(chromium), 2014/09/04 14:21:06]

logging accepts *args at the end, and does string formatting automatically only
if needed (i.e., logging level <= DEBUG);

logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)

will work faster on average.

[Sergiy Byelozyorov, 2014/09/05 12:59:06]

Done.

+  gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
+      gatekeeper_trees_path)
+
+  master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
   start_time = datetime.datetime.now()
 
   latest_builder_info = {}
 
   cache = buildbot.BuildCache(CACHE_PATH)
 
   alerts = []
   for master_url in master_urls:
     master_json = buildbot.fetch_master_json(master_url)
     master_alerts = alert_builder.alerts_for_master(cache,
         master_url, master_json, args.builder_filter)
     alerts.extend(master_alerts)
 
     # FIXME: This doesn't really belong here. garden-o-matic wants
     # this data and we happen to have the builder json cached at
     # this point so it's cheap to compute.
     builder_info = buildbot.latest_builder_info_for_master(cache,
         master_url, master_json)
     latest_builder_info.update(builder_info)
 
 
   print "Fetch took: %s" % (datetime.datetime.now() - start_time)
 
-  alerts = apply_gatekeeper_rules(alerts, gatekeeper)
+  alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
+                                                    gatekeeper_trees)
 
   alerts = analysis.assign_keys(alerts)
   reason_groups = analysis.group_by_reason(alerts)
   range_groups = analysis.merge_by_range(reason_groups)
   data = { 'content': json.dumps({
       'alerts': alerts,
       'reason_groups': reason_groups,
       'range_groups': range_groups,
       'latest_builder_info': latest_builder_info,
   })}
 
   if not args.data_url:
     with open('builder_alerts.json', 'w') as f:
       f.write(json.dumps(data, indent=1))
 
   for url in args.data_url:
     logging.info('POST %s alerts to %s' % (len(alerts), url))
     requests.post(url, data=data)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))