Add step generator protocol, remove annotated_checkout, remove script.

scripts/slave/recipe_util.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This module holds utilities which make writing recipes easier."""
 
 import contextlib as _contextlib
 import os as _os
 
+# These imports are intended to be passed through to recipes
+# pylint: disable=W0611
+from common.chromium_utils import IsWindows, IsMac, IsLinux
+
 # e.g. /b/build/slave/<slave-name>/build
 SLAVE_BUILD_ROOT = _os.path.abspath(_os.getcwd())
 # e.g. /b
 ROOT = _os.path.abspath(_os.path.join(SLAVE_BUILD_ROOT, _os.pardir, _os.pardir,
                                       _os.pardir, _os.pardir))
 # e.g. /b/build_internal
 BUILD_INTERNAL_ROOT = _os.path.join(ROOT, 'build_internal')
 # e.g. /b/build
 BUILD_ROOT = _os.path.join(ROOT, 'build')
 # e.g. /b/depot_tools
@@ skipping 183 matching lines @@
   parameters --factory-properties={...} and --build-properties={...} after
   your recipe terminates.
 
   Note that the 'steps' key will be absent from factory-properties. If you
   need to pass the list of steps to some of the steps, you will need to do
   that manually in your recipe (preferably with json.dumps()).
 
   This placeholder can be automatically added when you use the Steps.step()
   method in this module.
   """
-  pass
 PropertyPlaceholder = PropertyPlaceholder()
 
+class JsonOutputPlaceholder(object):
+  """JsonOutputPlaceholder is meant to be a singleton object which, when added
+  to a step's cmd list, will be replaced by annotated_run with the command
+  parameters --json-output=/path/to/file during the evaluation of your recipe
+  generator.
+
+  This placeholder can be optionally added when you use the Steps.step()
+  method in this module.
+
+  After the termination of the step, this file is expected to contain a valid
+  JSON document, which will be set as the json_output for that step in the
+  step_history OrderedDict passed to your recipe generator.
+  """
+JsonOutputPlaceholder = JsonOutputPlaceholder()
+
 
 def _url_method(name):
   """Returns a shortcut static method which functions like os.path.join and uses
   a fixed first url component which is chosen from the urls defined in
   SOURCE_URLS based on |name|.
   """
   # note that we do the __name__ munging for each function separately because
   # staticmethod hides these attributes.
   bases = SOURCE_URLS[name]
   if len(bases) == 1:
@@ skipping 24 matching lines @@
   def __init__(self, build_properties):
     self.build_properties = build_properties
     self.use_mirror = self.build_properties.get('use_mirror', True)
 
   def mirror_only(self, obj):
     """Returns obj if we're using mirrors. Otherwise returns the 'empty'
     version of obj.
     """
     return obj if self.use_mirror else obj.__class__()
 
-  def gclient_common_spec(self, solution_name):
-    """Returns a single gclient solution object (python dict) for common
-    solutions.
-    """
-    return GCLIENT_COMMON_SPECS[solution_name](self)
-
   @staticmethod
-  def step(name, cmd, add_properties=False, **kwargs):
+  def step(name, cmd, add_properties=False, add_json_output=False, **kwargs):
     """Returns a step dictionary which is compatible with annotator.py.
 
     Uses PropertyPlaceholder as a stand-in for build-properties and
     factory-properties so that annotated_run can fill them in after the recipe
     completes.
 
     Args:
       name: The name of this step.
       cmd: A list of strings in the style of subprocess.Popen.
       add_properties: Add PropertyPlaceholder iff True
+      add_json_output: Add JsonOutputPlaceholder iff True
       **kwargs: Additional entries to add to the annotator.py step dictionary.
 
     Returns:
       A step dictionary which is compatible with annotator.py.
     """
     assert 'shell' not in kwargs
     assert isinstance(cmd, list)
     if add_properties:
       cmd += [PropertyPlaceholder]
+    if add_json_output:
+      cmd += [JsonOutputPlaceholder]
     ret = kwargs
     ret.update({'name': name, 'cmd': cmd})
     return ret
 
-  def apply_issue_step(self, *root_pieces):
+  def apply_issue(self, *root_pieces):
     return self.step('apply_issue', [
         depot_tools_path('apply_issue'),
         '-r', checkout_path(*root_pieces),
         '-i', self.build_properties['issue'],
         '-p', self.build_properties['patchset'],
         '-s', self.build_properties['rietveld'],
         '-e', 'commit-bot@chromium.org'])
 
-  def git_step(self, *args):
+  def git(self, *args, **kwargs):
     name = 'git '+args[0]
     # Distinguish 'git config' commands by the variable they are setting.
     if args[0] == 'config' and not args[1].startswith('-'):
       name += ' ' + args[1]
     return self.step(name, [
         'git', '--work-tree', checkout_path(),
-        '--git-dir', checkout_path('.git')] + list(args))
+        '--git-dir', checkout_path('.git')] + list(args), **kwargs)
+
+  def generator_script(self, path_to_script):
+    def step_generator(step_history, _failure):
+      yield self.step(
+        'gen step(%s)' % _os.path.basename(path_to_script),
+        [path_to_script],
+        add_properties=True,
+        add_json_output=True,
+        cwd=checkout_path())
+      new_steps = step_history.last_step().json_data
+      assert isinstance(new_steps, list)
+      yield new_steps
+    return step_generator
+
+  def git_checkout(self, url, dir_path=None, branch='master', recursive=False):
+    if not dir_path:
+      dir_path = url.rsplit('/', 1)[-1]
+      if dir_path.endswith('.git'):  # ex: https://host/foobar.git
+        dir_path = dir_path[:-len('.git')]
+      if not dir_path:  # ex: ssh://host:repo/foobar/.git
+        dir_path = dir_path.rsplit('/', 1)[-1]
+      dir_path = slave_build_path(dir_path)
+    assert _os.pardir not in dir_path
+    recursive_args = ['--recurse-submodules'] if recursive else []
+    return [
+      self.step(
+        'git setup', [
+          build_path('scripts', 'slave', 'git_setup.py'),
+          '--path', dir_path,
+          '--url', url,
+        ],
+        static_json_data={
+          'CheckoutRoot': dir_path,
+          'CheckoutSCM': 'git',
+          'CheckoutSpec': {
+            'url': url,
+            'recursive': recursive,
+          },
+        }),
+      self.git('fetch', 'origin', *recursive_args),
+      self.git('update-ref', 'refs/heads/'+branch, 'origin/'+branch),
+      self.git('clean', '-f', '-d', '-X'),
+      self.git('checkout', '-f', branch),
+      self.git('submodule', 'update', '--init', '--recursive'),
+    ]
+
+  def gclient_checkout(self, common_repo_name, git_mode=False):
+    """Returns a step generator function for gclient checkouts."""
+    spec = GCLIENT_COMMON_SPECS[common_repo_name](self)
+    spec_string = ''
+    for key in spec:
+      # We should be using json.dumps here, but gclient directly execs the dict
+      # that it receives as the argument to --spec, so we have to have True,
+      # False, and None instead of JSON's true, false, and null.
+      spec_string += '%s = %s\n' % (key, str(spec[key]))
+    gclient = depot_tools_path('gclient') + ('.bat' if IsWindows() else '')
+
+    if not git_mode:
+      clean_step = self.step('gclient clean', [gclient, 'revert', '--nohooks'])
+      sync_step = self.step('gclient sync', [gclient, 'sync', '--nohooks'])
+    else:
+      # clean() isn't used because the gclient sync flags passed in checkout()
+      # do much the same thing, and they're more correct than doing a separate
+      # 'gclient revert' because it makes sure the other args are correct when
+      # a repo was deleted and needs to be re-cloned (notably
+      # --with_branch_heads), whereas 'revert' uses default args for clone
+      # operations.
+      #
+      # TODO(mmoss): To be like current official builders, this step could just
+      # delete the whole <slave_name>/build/ directory and start each build
+      # from scratch. That might be the least bad solution, at least until we
+      # have a reliable gclient method to produce a pristine working dir for
+      # git-based builds (e.g. maybe some combination of 'git reset/clean -fx'
+      # and removing the 'out' directory).
+      clean_step = None
+      sync_step = self.step('gclient sync', [
+        gclient, 'sync', '--verbose', '--with_branch_heads', '--nohooks',
+                     '--reset', '--delete_unversioned_trees', '--force'])
+    steps = [
+      self.step(
+        'gclient setup',
+        [gclient, 'config', '--spec', spec_string],
+        static_json_data={
+          'CheckoutRoot': slave_build_path(spec['solutions'][0]['name']),
+          'CheckoutSCM': 'gclient',
+          'CheckoutSpec': spec
+        }
+      ),
+    ]
+    if clean_step:
+      steps.append(clean_step)
+    if sync_step:
+      steps.append(sync_step)
+
+    return steps