Rework gclient 'recurse' command to use a WorkQueue.

gclient.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Meta checkout manager supporting both Subversion and GIT.
 
 Files
   .gclient      : Current client configuration, written by 'config' command.
                   Format is a Python script defining 'solutions', a list whose
@@ skipping 547 matching lines @@
       for i in range(len(file_list)):
         # It depends on the command being executed (like runhooks vs sync).
         if not os.path.isabs(file_list[i]):
           continue
         prefix = os.path.commonprefix(
             [self.root.root_dir.lower(), file_list[i].lower()])
         file_list[i] = file_list[i][len(prefix):]
         # Strip any leading path separators.
         while file_list[i].startswith(('\\', '/')):
           file_list[i] = file_list[i][1:]
+    elif command is None and args:
+      # Used by CMDrecurse to run an arbitrary command
+      if not isinstance(parsed_url, self.FileImpl):
+        # Skip file only checkout.
+        scm = gclient_scm.GetScmName(parsed_url)
+        if not options.scm or scm in options.scm:
+          cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
+          # Pass in the SCM type as an env variable
+          env = os.environ.copy()
+          if scm:
+            env['GCLIENT_SCM'] = scm
+          if parsed_url:
+            env['GCLIENT_URL'] = parsed_url
+          if os.path.isdir(cwd):
+            gclient_utils.CheckCallAndFilterAndHeader(args, cwd=cwd, env=env)

[davidbarr, 2012/03/02 01:23:05]

Now the output is handled nicely.

+          else:
+            print >> sys.stderr, 'Skipped missing %s' % cwd
 
     # Always parse the DEPS file.
     self.ParseDepsFile()
 
     self._run_is_done(file_list, parsed_url)
 
     if self.recursion_limit:
       # Parse the dependencies of this dependency.
       for s in self.dependencies:
         work_queue.enqueue(s)
@@ skipping 546 matching lines @@
 @attr('usage', '[command] [args ...]')
 def CMDrecurse(parser, args):
   """Operates on all the entries.
 
   Runs a shell command on all entries.
   """
   # Stop parsing at the first non-arg so that these go through to the command
   parser.disable_interspersed_args()
   parser.add_option('-s', '--scm', action='append', default=[],
                     help='choose scm types to operate upon')
-  parser.remove_option('--jobs')
   options, args = parser.parse_args(args)
   if not args:
     print >> sys.stderr, 'Need to supply a command!'
     return 1
   root_and_entries = gclient_utils.GetGClientRootAndEntries()
   if not root_and_entries:
     print >> sys.stderr, (
         'You need to run gclient sync at least once to use \'recurse\'.\n'
         'This is because .gclient_entries needs to exist and be up to date.')
     return 1
-  root, entries = root_and_entries
+
+  # Normalize options.scm to a set()
   scm_set = set()
   for scm in options.scm:
     scm_set.update(scm.split(','))
+  options.scm = scm_set
 
-  # Pass in the SCM type as an env variable
-  env = os.environ.copy()
-
-  for path, url in entries.iteritems():
-    scm = gclient_scm.GetScmName(url)
-    if scm_set and scm not in scm_set:
-      continue
-    cwd = os.path.normpath(os.path.join(root, path))
-    if scm:
-      env['GCLIENT_SCM'] = scm
-    if url:
-      env['GCLIENT_URL'] = url
-    if os.path.isdir(cwd):
-      subprocess2.call(args, cwd=cwd, env=env)
-    else:
-      print >> sys.stderr, 'Skipped missing %s' % cwd
+  client = GClient.LoadCurrentConfig(options)
+  pm = None

[M-A Ruel, 2012/03/02 01:40:37]

Replace lines 1168-1176 with:
options.nohooks = True
client = GClient.LoadCurrentConfig(options)
return client.RunOnDeps(None, args)

In theory, it should just work. :) In practice, you'll want to tweak the
Progress object. I'm fine with an additional argument to RunOnDeps().

+  # Disable progress for non-tty stdout.
+  if (sys.stdout.isatty() and not options.verbose):
+    pm = Progress(' '.join(args), 1)
+  work_queue = gclient_utils.ExecutionQueue(options.jobs, pm)
+  for s in client.dependencies:
+    work_queue.enqueue(s)
+  work_queue.flush({}, None, args, options=options)
   return 0
 
 
 @attr('usage', '[args ...]')
 def CMDfetch(parser, args):
   """Fetches upstream commits for all modules.
 
 Completely git-specific. Simply runs 'git fetch [args ...]' for each module.
 """
-  (_, args) = parser.parse_args(args)
-  args = ['-s', 'git', 'git', 'fetch'] + args
+  (options, args) = parser.parse_args(args)
+  args = ['-j%d' % options.jobs, '-s', 'git', 'git', 'fetch'] + args

[davidbarr, 2012/03/02 01:23:05]

This still makes me uncomfortable, but there isn't a cleaner means of performing
this mapping yet.

   return CMDrecurse(parser, args)
 
 
 @attr('usage', '[url] [safesync url]')
 def CMDconfig(parser, args):
   """Create a .gclient file in the current directory.
 
 This specifies the configuration for further commands. After update/sync,
 top-level DEPS files in each module are read to determine dependent
 modules to operate on as well. If optional [url] parameter is
@@ skipping 362 matching lines @@
   except (gclient_utils.Error, subprocess2.CalledProcessError), e:
     print >> sys.stderr, 'Error: %s' % str(e)
     return 1
 
 
 if '__main__' == __name__:
   fix_encoding.fix_encoding()
   sys.exit(Main(sys.argv[1:]))
 
 # vim: ts=2:sw=2:tw=80:et: