Rework gclient 'recurse' command to use a WorkQueue.

gclient.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Meta checkout manager supporting both Subversion and GIT.
 
 Files
   .gclient      : Current client configuration, written by 'config' command.
                   Format is a Python script defining 'solutions', a list whose
@@ skipping 1103 matching lines @@
   client = GClient.LoadCurrentConfig(options)
   if not client:
     raise gclient_utils.Error('client not configured; see \'gclient config\'')
   if options.verbose:
     # Print out the .gclient file.  This is longer than if we just printed the
     # client dict, but more legible, and it might contain helpful comments.
     print(client.config_content)
   return client.RunOnDeps('cleanup', args)
 
 
+class Entry(gclient_utils.WorkItem):

[M-A Ruel, 2012/03/01 14:12:12]

I'm a tad uncomfortable with this class. I'd rather you to use the "normal way"
to use the work queue and change Dependency.run() to be able to run an arbitrary
functor. This means adding a new "command". I understand that it's more
refactoring but it's easier on the long run.

In particular, you could use command = None but use args. Right now "args" is
ignored with command == None. Just make sure to document the new behavior. In
particular, the diff would look like:


diff --git a/gclient.py b/gclient.py
index 9931ad7..49de5bf 100644
--- a/gclient.py
+++ b/gclient.py
@@ -565,6 +565,23 @@ class Dependency(gclient_utils.WorkItem,
DependencySettings):
         # Strip any leading path separators.
         while file_list[i].startswith(('\\', '/')):
           file_list[i] = file_list[i][1:]
+    elif command is None and args:
+      # Used by CMDrecurse to run an arbitrary command.
+      if not instance(parsed_url, self.FileImpl):
+        # Skip file only checkout.
+        scm = gclient_scm.GetScmName(parsed_url)
+        # TODO(davidbarr): Convert options.scm as a set()
+        if options.scm and scm not in options.scm:
+          continue
+        cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
+        if scm:
+          env['GCLIENT_SCM'] = scm
+        if parsed_url:
+          env['GCLIENT_URL'] = parsed_url
+        if os.path.isdir(cwd):
+          subprocess2.call(args, cwd=cwd, env=env)
+        else:
+          print >> sys.stderr, 'Skipped missing %s' % cwd

     # Always parse the DEPS file.
     self.ParseDepsFile()


Then you can remove the corresponding conditions in CMDrecurse.

+  """Object that represents a gclient entry."""
+
+  def __init__(self, name, scm, cwd, url):
+    gclient_utils.WorkItem.__init__(self, name)
+    self._scm = scm
+    self._cwd = cwd
+    self._url = url
+    self.requirements = {}
+
+  # Arguments number differs from overridden method
+  # pylint: disable=W0221
+  def run(self, revision_overrides, command, args, work_queue, options):
+    """Executes a command in a given module."""
+    assert command == 'exec'
+    # Pass in the SCM type as an env variable
+    env = os.environ.copy()
+    if self._scm:
+      env['GCLIENT_SCM'] = self._scm
+    if self._url:
+      env['GCLIENT_URL'] = self._url
+    if os.path.isdir(self._cwd):
+      subprocess2.call(args, cwd=self._cwd, env=env)

[davidbarr, 2012/03/01 05:33:15]

Maybe this should be a wrapper method provided by WorkQueue?

+    else:
+      print >> sys.stderr, 'Skipped missing %s' % self._cwd
+

[davidbarr, 2012/03/01 05:33:15]

Whitespace nit.

 @attr('usage', '[command] [args ...]')
 def CMDrecurse(parser, args):
   """Operates on all the entries.
 
   Runs a shell command on all entries.
   """
   # Stop parsing at the first non-arg so that these go through to the command
   parser.disable_interspersed_args()
   parser.add_option('-s', '--scm', action='append', default=[],
                     help='choose scm types to operate upon')
-  parser.remove_option('--jobs')
   options, args = parser.parse_args(args)
   if not args:
     print >> sys.stderr, 'Need to supply a command!'
     return 1
   root_and_entries = gclient_utils.GetGClientRootAndEntries()
   if not root_and_entries:
     print >> sys.stderr, (
         'You need to run gclient sync at least once to use \'recurse\'.\n'
         'This is because .gclient_entries needs to exist and be up to date.')
     return 1
   root, entries = root_and_entries
   scm_set = set()
   for scm in options.scm:
     scm_set.update(scm.split(','))
 
-  # Pass in the SCM type as an env variable
-  env = os.environ.copy()
-
+  pm = None
+  # Disable progress for non-tty stdout.
+  if (sys.stdout.isatty() and not options.verbose):
+    pm = Progress(' '.join(args), 1)
+  work_queue = gclient_utils.ExecutionQueue(options.jobs, pm)
   for path, url in entries.iteritems():
     scm = gclient_scm.GetScmName(url)
     if scm_set and scm not in scm_set:
       continue
     cwd = os.path.normpath(os.path.join(root, path))
-    if scm:
-      env['GCLIENT_SCM'] = scm
-    if url:
-      env['GCLIENT_URL'] = url
-    if os.path.isdir(cwd):
-      subprocess2.call(args, cwd=cwd, env=env)
-    else:
-      print >> sys.stderr, 'Skipped missing %s' % cwd
+    work_queue.enqueue(Entry(path, scm, cwd, url))
+  work_queue.flush({}, 'exec', args, options=options)
   return 0
 
 
 @attr('usage', '[args ...]')
 def CMDfetch(parser, args):
   """Fetches upstream commits for all modules.
 
 Completely git-specific. Simply runs 'git fetch [args ...]' for each module.
 """
-  (_, args) = parser.parse_args(args)
-  args = ['-s', 'git', 'git', 'fetch'] + args
+  (options, args) = parser.parse_args(args)
+  args = ['-j%d' % options.jobs, '-s', 'git', 'git', 'fetch'] + args

[davidbarr, 2012/03/01 05:33:15]

Is this necessary? Will the parser accumulate options when it is reinvoked?

   return CMDrecurse(parser, args)
 
 
 @attr('usage', '[url] [safesync url]')
 def CMDconfig(parser, args):
   """Create a .gclient file in the current directory.
 
 This specifies the configuration for further commands. After update/sync,
 top-level DEPS files in each module are read to determine dependent
 modules to operate on as well. If optional [url] parameter is
@@ skipping 362 matching lines @@
   except (gclient_utils.Error, subprocess2.CalledProcessError), e:
     print >> sys.stderr, 'Error: %s' % str(e)
     return 1
 
 
 if '__main__' == __name__:
   fix_encoding.fix_encoding()
   sys.exit(Main(sys.argv[1:]))
 
 # vim: ts=2:sw=2:tw=80:et: