Rework gclient 'recurse' command to use a WorkQueue.

gclient.py

Index: gclient.py
===================================================================
--- gclient.py	(revision 124337)
+++ gclient.py	(working copy)
@@ -1121,6 +1121,32 @@
   return client.RunOnDeps('cleanup', args)
 
 
+class Entry(gclient_utils.WorkItem):

[M-A Ruel, 2012/03/01 14:12:12]

I'm a tad uncomfortable with this class. I'd rather you to use the "normal way"
to use the work queue and change Dependency.run() to be able to run an arbitrary
functor. This means adding a new "command". I understand that it's more
refactoring but it's easier on the long run.

In particular, you could use command = None but use args. Right now "args" is
ignored with command == None. Just make sure to document the new behavior. In
particular, the diff would look like:


diff --git a/gclient.py b/gclient.py
index 9931ad7..49de5bf 100644
--- a/gclient.py
+++ b/gclient.py
@@ -565,6 +565,23 @@ class Dependency(gclient_utils.WorkItem,
DependencySettings):
         # Strip any leading path separators.
         while file_list[i].startswith(('\\', '/')):
           file_list[i] = file_list[i][1:]
+    elif command is None and args:
+      # Used by CMDrecurse to run an arbitrary command.
+      if not instance(parsed_url, self.FileImpl):
+        # Skip file only checkout.
+        scm = gclient_scm.GetScmName(parsed_url)
+        # TODO(davidbarr): Convert options.scm as a set()
+        if options.scm and scm not in options.scm:
+          continue
+        cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
+        if scm:
+          env['GCLIENT_SCM'] = scm
+        if parsed_url:
+          env['GCLIENT_URL'] = parsed_url
+        if os.path.isdir(cwd):
+          subprocess2.call(args, cwd=cwd, env=env)
+        else:
+          print >> sys.stderr, 'Skipped missing %s' % cwd

     # Always parse the DEPS file.
     self.ParseDepsFile()


Then you can remove the corresponding conditions in CMDrecurse.

+  """Object that represents a gclient entry."""
+
+  def __init__(self, name, scm, cwd, url):
+    gclient_utils.WorkItem.__init__(self, name)
+    self._scm = scm
+    self._cwd = cwd
+    self._url = url
+    self.requirements = {}
+
+  # Arguments number differs from overridden method
+  # pylint: disable=W0221
+  def run(self, revision_overrides, command, args, work_queue, options):
+    """Executes a command in a given module."""
+    assert command == 'exec'
+    # Pass in the SCM type as an env variable
+    env = os.environ.copy()
+    if self._scm:
+      env['GCLIENT_SCM'] = self._scm
+    if self._url:
+      env['GCLIENT_URL'] = self._url
+    if os.path.isdir(self._cwd):
+      subprocess2.call(args, cwd=self._cwd, env=env)

[davidbarr, 2012/03/01 05:33:15]

Maybe this should be a wrapper method provided by WorkQueue?

+    else:
+      print >> sys.stderr, 'Skipped missing %s' % self._cwd
+

[davidbarr, 2012/03/01 05:33:15]

Whitespace nit.

 @attr('usage', '[command] [args ...]')
 def CMDrecurse(parser, args):
   """Operates on all the entries.
@@ -1131,7 +1157,6 @@
   parser.disable_interspersed_args()
   parser.add_option('-s', '--scm', action='append', default=[],
                     help='choose scm types to operate upon')
-  parser.remove_option('--jobs')
   options, args = parser.parse_args(args)
   if not args:
     print >> sys.stderr, 'Need to supply a command!'
@@ -1147,22 +1172,18 @@
   for scm in options.scm:
     scm_set.update(scm.split(','))
 
-  # Pass in the SCM type as an env variable
-  env = os.environ.copy()
-
+  pm = None
+  # Disable progress for non-tty stdout.
+  if (sys.stdout.isatty() and not options.verbose):
+    pm = Progress(' '.join(args), 1)
+  work_queue = gclient_utils.ExecutionQueue(options.jobs, pm)
   for path, url in entries.iteritems():
     scm = gclient_scm.GetScmName(url)
     if scm_set and scm not in scm_set:
       continue
     cwd = os.path.normpath(os.path.join(root, path))
-    if scm:
-      env['GCLIENT_SCM'] = scm
-    if url:
-      env['GCLIENT_URL'] = url
-    if os.path.isdir(cwd):
-      subprocess2.call(args, cwd=cwd, env=env)
-    else:
-      print >> sys.stderr, 'Skipped missing %s' % cwd
+    work_queue.enqueue(Entry(path, scm, cwd, url))
+  work_queue.flush({}, 'exec', args, options=options)
   return 0
 
 
@@ -1172,8 +1193,8 @@
 
 Completely git-specific. Simply runs 'git fetch [args ...]' for each module.
 """
-  (_, args) = parser.parse_args(args)
-  args = ['-s', 'git', 'git', 'fetch'] + args
+  (options, args) = parser.parse_args(args)
+  args = ['-j%d' % options.jobs, '-s', 'git', 'git', 'fetch'] + args

[davidbarr, 2012/03/01 05:33:15]

Is this necessary? Will the parser accumulate options when it is reinvoked?

   return CMDrecurse(parser, args)