Add step generator protocol, remove annotated_checkout, remove script.

scripts/slave/recipe_util.py

Index: scripts/slave/recipe_util.py
diff --git a/scripts/slave/recipe_util.py b/scripts/slave/recipe_util.py
index 4a00ef9ceb0365840ec187bffd3c62ff792886ed..dd9cff57512fee0c9653a67334082ba9fb07f585 100644
--- a/scripts/slave/recipe_util.py
+++ b/scripts/slave/recipe_util.py
@@ -7,6 +7,10 @@
 import contextlib as _contextlib
 import os as _os
 
+# These imports are intended to be passed through to recipes
+# pylint: disable=W0611
+from common.chromium_utils import IsWindows, IsMac, IsLinux
+
 # e.g. /b/build/slave/<slave-name>/build
 SLAVE_BUILD_ROOT = _os.path.abspath(_os.getcwd())
 # e.g. /b
@@ -210,9 +214,23 @@ class PropertyPlaceholder(object):
   This placeholder can be automatically added when you use the Steps.step()
   method in this module.
   """
-  pass
 PropertyPlaceholder = PropertyPlaceholder()
 
+class JsonOutputPlaceholder(object):
+  """JsonOutputPlaceholder is meant to be a singleton object which, when added
+  to a step's cmd list, will be replaced by annotated_run with the command
+  parameters --json-output=/path/to/file during the evaluation of your recipe
+  generator.
+
+  This placeholder can be optionally added when you use the Steps.step()
+  method in this module.
+
+  After the termination of the step, this file is expected to contain a valid
+  JSON document, which will be set as the json_output for that step in the
+  step_history OrderedDict passed to your recipe generator.
+  """
+JsonOutputPlaceholder = JsonOutputPlaceholder()
+
 
 def _url_method(name):
   """Returns a shortcut static method which functions like os.path.join and uses
@@ -257,14 +275,8 @@ class Steps(object):
     """
     return obj if self.use_mirror else obj.__class__()
 
-  def gclient_common_spec(self, solution_name):
-    """Returns a single gclient solution object (python dict) for common
-    solutions.
-    """
-    return GCLIENT_COMMON_SPECS[solution_name](self)
-
   @staticmethod
-  def step(name, cmd, add_properties=False, **kwargs):
+  def step(name, cmd, add_properties=False, add_json_output=False, **kwargs):
     """Returns a step dictionary which is compatible with annotator.py.
 
     Uses PropertyPlaceholder as a stand-in for build-properties and
@@ -275,6 +287,7 @@ class Steps(object):
       name: The name of this step.
       cmd: A list of strings in the style of subprocess.Popen.
       add_properties: Add PropertyPlaceholder iff True
+      add_json_output: Add JsonOutputPlaceholder iff True
       **kwargs: Additional entries to add to the annotator.py step dictionary.
 
     Returns:
@@ -284,11 +297,13 @@ class Steps(object):
     assert isinstance(cmd, list)
     if add_properties:
       cmd += [PropertyPlaceholder]
+    if add_json_output:
+      cmd += [JsonOutputPlaceholder]
     ret = kwargs
     ret.update({'name': name, 'cmd': cmd})
     return ret
 
-  def apply_issue_step(self, *root_pieces):
+  def apply_issue(self, *root_pieces):
     return self.step('apply_issue', [
         depot_tools_path('apply_issue'),
         '-r', checkout_path(*root_pieces),
@@ -297,11 +312,106 @@ class Steps(object):
         '-s', self.build_properties['rietveld'],
         '-e', 'commit-bot@chromium.org'])
 
-  def git_step(self, *args):
+  def git(self, *args, **kwargs):
     name = 'git '+args[0]
     # Distinguish 'git config' commands by the variable they are setting.
     if args[0] == 'config' and not args[1].startswith('-'):
       name += ' ' + args[1]
     return self.step(name, [
         'git', '--work-tree', checkout_path(),
-        '--git-dir', checkout_path('.git')] + list(args))
+        '--git-dir', checkout_path('.git')] + list(args), **kwargs)
+
+  def generator_script(self, path_to_script):
+    def step_generator(step_history, _failure):
+      yield self.step(
+        'gen step(%s)' % _os.path.basename(path_to_script),
+        [path_to_script],
+        add_properties=True,
+        add_json_output=True,
+        cwd=checkout_path())
+      new_steps = step_history.last_step().json_data
+      assert isinstance(new_steps, list)
+      yield new_steps
+    return step_generator
+
+  def git_checkout(self, url, dir_path=None, branch='master', recursive=False):
+    if not dir_path:
+      dir_path = url.rsplit('/', 1)[-1]
+      if dir_path.endswith('.git'):  # ex: https://host/foobar.git
+        dir_path = dir_path[:-len('.git')]
+      if not dir_path:  # ex: ssh://host:repo/foobar/.git
+        dir_path = dir_path.rsplit('/', 1)[-1]
+      dir_path = slave_build_path(dir_path)
+    assert _os.pardir not in dir_path
+    recursive_args = ['--recurse-submodules'] if recursive else []
+    return [
+      self.step(
+        'git setup', [
+          build_path('scripts', 'slave', 'git_setup.py'),
+          '--path', dir_path,
+          '--url', url,
+        ],
+        static_json_data={
+          'CheckoutRoot': dir_path,
+          'CheckoutSCM': 'git',
+          'CheckoutSpec': {
+            'url': url,
+            'recursive': recursive,
+          },
+        }),
+      self.git('fetch', 'origin', *recursive_args),
+      self.git('update-ref', 'refs/heads/'+branch, 'origin/'+branch),
+      self.git('clean', '-f', '-d', '-X'),
+      self.git('checkout', '-f', branch),
+      self.git('submodule', 'update', '--init', '--recursive'),
+    ]
+
+  def gclient_checkout(self, common_repo_name, git_mode=False):
+    """Returns a step generator function for gclient checkouts."""
+    spec = GCLIENT_COMMON_SPECS[common_repo_name](self)
+    spec_string = ''
+    for key in spec:
+      # We should be using json.dumps here, but gclient directly execs the dict
+      # that it receives as the argument to --spec, so we have to have True,
+      # False, and None instead of JSON's true, false, and null.
+      spec_string += '%s = %s\n' % (key, str(spec[key]))
+    gclient = depot_tools_path('gclient') + ('.bat' if IsWindows() else '')
+
+    if not git_mode:
+      clean_step = self.step('gclient clean', [gclient, 'revert', '--nohooks'])
+      sync_step = self.step('gclient sync', [gclient, 'sync', '--nohooks'])
+    else:
+      # clean() isn't used because the gclient sync flags passed in checkout()
+      # do much the same thing, and they're more correct than doing a separate
+      # 'gclient revert' because it makes sure the other args are correct when
+      # a repo was deleted and needs to be re-cloned (notably
+      # --with_branch_heads), whereas 'revert' uses default args for clone
+      # operations.
+      #
+      # TODO(mmoss): To be like current official builders, this step could just
+      # delete the whole <slave_name>/build/ directory and start each build
+      # from scratch. That might be the least bad solution, at least until we
+      # have a reliable gclient method to produce a pristine working dir for
+      # git-based builds (e.g. maybe some combination of 'git reset/clean -fx'
+      # and removing the 'out' directory).
+      clean_step = None
+      sync_step = self.step('gclient sync', [
+        gclient, 'sync', '--verbose', '--with_branch_heads', '--nohooks',
+                     '--reset', '--delete_unversioned_trees', '--force'])
+    steps = [
+      self.step(
+        'gclient setup',
+        [gclient, 'config', '--spec', spec_string],
+        static_json_data={
+          'CheckoutRoot': slave_build_path(spec['solutions'][0]['name']),
+          'CheckoutSCM': 'gclient',
+          'CheckoutSpec': spec
+        }
+      ),
+    ]
+    if clean_step:
+      steps.append(clean_step)
+    if sync_step:
+      steps.append(sync_step)
+
+    return steps