Support multiple try masters when sending tries to rietveld.

presubmit_support.py

Index: presubmit_support.py
diff --git a/presubmit_support.py b/presubmit_support.py
index dabfbf0bac2e7bbf253aaa29245fca5378ed58eb..3099ce84d60faea6f1ce64a2d6c66a6c12f0c135 100755
--- a/presubmit_support.py
+++ b/presubmit_support.py
@@ -19,6 +19,7 @@ import contextlib
 import fnmatch
 import glob
 import inspect
+import itertools
 import json  # Exposed through the API.
 import logging
 import marshal  # Exposed through the API.
@@ -1007,6 +1008,8 @@ class GetTrySlavesExecuter(object):
   @staticmethod
   def ExecPresubmitScript(script_text, presubmit_path, project, change):
     """Executes GetPreferredTrySlaves() from a single presubmit script.
+    
+    This will soon be deprecated and replaced by GetPreferredTryMasters().
 
     Args:
       script_text: The text of the presubmit script.
@@ -1074,6 +1077,68 @@ class GetTrySlavesExecuter(object):
     return result
 
 
+class GetTryMastersExecuter(object):
+  @staticmethod
+  def ExecPresubmitScript(script_text, presubmit_path, project, change):
+    """Executes GetPreferredTryMasters() from a single presubmit script.
+
+    Args:
+      script_text: The text of the presubmit script.
+      presubmit_path: Project script to run.
+      project: Project name to pass to presubmit script for bot selection.
+
+    Return:
+      A map of try masters to map of builders to set of tests.
+    """
+    context = {}
+    try:
+      exec script_text in context

[ghost stip (do not use), 2014/03/01 00:56:16]

if possible, try to refactor this out to share code with getprefeeredtryslaves.
ok for a separate cl

[Michael Achenbach, 2014/03/01 01:30:40]

Follow up CL.

+    except Exception, e:
+      raise PresubmitFailure('"%s" had an exception.\n%s'
+                             % (presubmit_path, e))
+
+    function_name = 'GetPreferredTryMasters'
+    if function_name not in context:
+      return {}
+    get_preferred_try_masters = context[function_name]
+    if not len(inspect.getargspec(get_preferred_try_masters)[0]) == 2:
+      raise PresubmitFailure(
+          'Expected function "GetPreferredTryMasters" to take two arguments.')
+    result = get_preferred_try_masters(project, change)
+
+    def CheckType(item, t):

[ghost stip (do not use), 2014/03/01 00:56:16]

I'm ok removing lines 1109 to 1141, since any code which doesn't have this
format will break

[Michael Achenbach, 2014/03/01 01:30:40]

Done.

+      if not isinstance(item, t):
+        raise PresubmitFailure(
+            'Expected a %s, got a %s instead: %s' %
+            (str(t), type(item), str(item)))
+
+    def CheckString(item):
+      if not item:
+        raise PresubmitFailure(
+            'PRESUBMIT.py returned empty trymaster/builder string.')
+      CheckType(item, basestring)
+      if item != item.strip():
+        raise PresubmitFailure(
+            'Try master/builder names cannot start/end with whitespace')
+      if ',' in item:
+        raise PresubmitFailure(
+            'Do not use \',\' separated master, builder or test names: %s'
+            % item)
+
+    CheckType(result, dict)
+    for (master, builders) in result.iteritems():
+      CheckString(master)
+      CheckType(builders, dict)
+      for (builder, tests) in builders.iteritems():
+        CheckString(builder)
+        CheckType(tests, set)
+        if not tests:
+          raise PresubmitFailure(
+            'PRESUBMIT.py returned empty tests for builder %s. '
+            'Maybe specify "defaulttests"?' % builder)
+    return result
+
+
 def DoGetTrySlaves(change,
                    changed_files,
                    repository_root,
@@ -1081,7 +1146,7 @@ def DoGetTrySlaves(change,
                    project,
                    verbose,
                    output_stream):
-  """Get the list of try servers from the presubmit scripts.
+  """Get the list of try servers from the presubmit scripts (deprecated).
 
   Args:
     changed_files: List of modified files.
@@ -1132,6 +1197,69 @@ def DoGetTrySlaves(change,
   return slaves
 
 
+def _MergeMasters(masters1, masters2):
+  """Merges two master maps. Merges also the tests of each builder.
+  """

[ghost stip (do not use), 2014/03/01 00:56:16]

nit: keep """ on same line

+  result = {}
+  for (master, builders) in itertools.chain(masters1.iteritems(),
+                                            masters2.iteritems()):
+    new_builders = result.setdefault(master, {})
+    for (builder, tests) in builders.iteritems():
+      new_builders.setdefault(builder, set([])).update(tests)
+  return result
+
+
+def DoGetTryMasters(change,
+                    changed_files,
+                    repository_root,
+                    default_presubmit,
+                    project,
+                    verbose,
+                    output_stream):
+  """Get the list of try masters from the presubmit scripts.
+
+  Args:
+    changed_files: List of modified files.
+    repository_root: The repository root.
+    default_presubmit: A default presubmit script to execute in any case.
+    project: Optional name of a project used in selecting trybots.
+    verbose: Prints debug info.
+    output_stream: A stream to write debug output to.
+
+  Return:
+    Map of try masters to map of builders to set of tests.
+  """
+  presubmit_files = ListRelevantPresubmitFiles(changed_files, repository_root)
+  if not presubmit_files and verbose:
+    output_stream.write("Warning, no presubmit.py found.\n")

[ghost stip (do not use), 2014/03/01 00:56:16]

nit: PRESUBMIT.py vs presubmit.py

[Michael Achenbach, 2014/03/01 01:30:40]

Done.

+  results = {}
+  executer = GetTryMastersExecuter()
+
+  if default_presubmit:
+    if verbose:
+      output_stream.write("Running default presubmit script.\n")
+    fake_path = os.path.join(repository_root, 'PRESUBMIT.py')
+    results = _MergeMasters(results, executer.ExecPresubmitScript(
+        default_presubmit, fake_path, project, change))
+  for filename in presubmit_files:
+    filename = os.path.abspath(filename)
+    if verbose:
+      output_stream.write("Running %s\n" % filename)
+    # Accept CRLF presubmit script.
+    presubmit_script = gclient_utils.FileRead(filename, 'rU')
+    results = _MergeMasters(results, executer.ExecPresubmitScript(
+        presubmit_script, filename, project, change))
+
+  # Make sets to lists again for later JSON serialization.
+  for (_, builders) in results.iteritems():

[ghost stip (do not use), 2014/03/01 00:56:16]

for builders in results.itervalues()

[Michael Achenbach, 2014/03/01 01:30:40]

Done.

+    for builder in builders:
+      builders[builder] = list(builders[builder])
+
+  if results and verbose:
+    output_stream.write('%s\n' % str(results))
+  return results
+
+
 class PresubmitExecuter(object):
   def __init__(self, change, committing, rietveld_obj, verbose):
     """