Add gclient grep for git repos

gclient.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Meta checkout manager supporting both Subversion and GIT.
 
 Files
   .gclient      : Current client configuration, written by 'config' command.
                   Format is a Python script defining 'solutions', a list whose
@@ skipping 551 matching lines @@
         not gclient_utils.IsDateRevision(options.revision)):
       revision_date = scm.GetRevisionDate(options.revision)
       revision = gclient_utils.MakeDateRevision(revision_date)
       if options.verbose:
         print("Updating revision override from %s to %s." %
               (options.revision, revision))
       revision_overrides[name] = revision
 
   # Arguments number differs from overridden method
   # pylint: disable=W0221
-  def run(self, revision_overrides, command, args, work_queue, options):
+  def run(self, revision_overrides, command, args, work_queue, options,
+          **kwargs):
     """Runs |command| then parse the DEPS file."""
     logging.info('Dependency(%s).run()' % self.name)
     assert self._file_list == []
     if not self.should_process:
       return
     # When running runhooks, there's no need to consult the SCM.
     # All known hooks are expected to run unconditionally regardless of working
     # copy state, so skip the SCM status check.
     run_scm = command not in ('runhooks', 'recurse', None)
     parsed_url = self.LateOverride(self.url)
@@ skipping 28 matching lines @@
       for i in range(len(file_list)):
         # It depends on the command being executed (like runhooks vs sync).
         if not os.path.isabs(file_list[i]):
           continue
         prefix = os.path.commonprefix(
             [self.root.root_dir.lower(), file_list[i].lower()])
         file_list[i] = file_list[i][len(prefix):]
         # Strip any leading path separators.
         while file_list[i].startswith(('\\', '/')):
           file_list[i] = file_list[i][1:]
-    elif command == 'recurse':
+
+    # Always parse the DEPS file.
+    self.ParseDepsFile()
+
+    self._run_is_done(file_list, parsed_url)
+
+    if self.recursion_limit:
+      # Parse the dependencies of this dependency.
+      for s in self.dependencies:
+        work_queue.enqueue(s)
+
+    if command == 'recurse':
       if not isinstance(parsed_url, self.FileImpl):
         # Skip file only checkout.
         scm = gclient_scm.GetScmName(parsed_url)
         if not options.scm or scm in options.scm:
           cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
           # Pass in the SCM type as an env variable
           env = os.environ.copy()
           if scm:
             env['GCLIENT_SCM'] = scm
           if parsed_url:
             env['GCLIENT_URL'] = parsed_url
+          print_stdout = True
+          filter_fn = None
+          if options.prepend_dir:
+            print_stdout = False
+            def filter_fn(line):
+              items = line.split('\0')

[M-A Ruel, 2012/11/08 19:44:13]

if '\0' in line:
...
else:
  items = line.split('\0', 1)
  print ':'.join(os.path.join(self.name, item[0]), item[1])

and ignore any remaining null if any.

[Isaac (away), 2012/11/09 03:58:52]

I'm confused -- what are you suggesting here?  Will it cover the same use cases
as I have below?

The else clause is needed for git grep --null -l where git does not print
newlines, but instead separates multiple matchs with just a nullbyte.

+              if len(items) == 1:
+                match = re.match('Binary file (.*) matches$', line)
+                if match:
+                  print 'Binary file %s matches' % os.path.join(
+                      self.name, match.group(1))
+                else:
+                  print line
+              elif len(items) == 2 and items[1]:
+                print '%s : %s' % (os.path.join(self.name, items[0]), items[1])
+              else:
+                # Multiple null bytes or a single trailing null byte indicate
+                # git is likely displaying filenames only (such as with -l)
+                print '\n'.join(os.path.join(self.name, f) for f in items if f)
           if os.path.isdir(cwd):
             try:
               gclient_utils.CheckCallAndFilter(
-                  args, cwd=cwd, env=env, print_stdout=True)
+                  args, cwd=cwd, env=env, print_stdout=print_stdout,
+                  filter_fn=filter_fn,
+                  )
             except subprocess2.CalledProcessError:
               if not options.ignore:
                 raise
           else:
             print >> sys.stderr, 'Skipped missing %s' % cwd
 
-    # Always parse the DEPS file.
-    self.ParseDepsFile()
-
-    self._run_is_done(file_list, parsed_url)
-
-    if self.recursion_limit:
-      # Parse the dependencies of this dependency.
-      for s in self.dependencies:
-        work_queue.enqueue(s)
 
   @gclient_utils.lockedmethod
   def _run_is_done(self, file_list, parsed_url):
     # Both these are kept for hooks that are run as a separate tree traversal.
     self._file_list = file_list
     self._parsed_url = parsed_url
     self._processed = True
 
   @staticmethod
   def GetHookAction(hook_dict, matching_file_list):
@@ skipping 381 matching lines @@
       raise gclient_utils.Error(
           'It appears your safesync_url (%s) is not working properly\n'
           '(as it returned an empty response). Check your config.' %
           dep.safesync_url)
     scm = gclient_scm.CreateSCM(dep.url, dep.root.root_dir, dep.name)
     safe_rev = scm.GetUsableRev(rev=rev, options=self._options)
     if self._options.verbose:
       print('Using safesync_url revision: %s.\n' % safe_rev)
     self._options.revisions.append('%s@%s' % (dep.name, safe_rev))
 
-  def RunOnDeps(self, command, args):
+  def RunOnDeps(self, command, args, ignore_requirements=False):
     """Runs a command on each dependency in a client and its dependencies.
 
     Args:
       command: The command to use (e.g., 'status' or 'diff')
       args: list of str - extra arguments to add to the command line.
     """
     if not self.dependencies:
       raise gclient_utils.Error('No solution specified')
     revision_overrides = {}
     # It's unnecessary to check for revision overrides for 'recurse'.
     # Save a few seconds by not calling _EnforceRevisions() in that case.
     if command not in ('diff', 'recurse', 'runhooks', 'status'):
       revision_overrides = self._EnforceRevisions()
     pm = None
     # Disable progress for non-tty stdout.
     if (sys.stdout.isatty() and not self._options.verbose):
       if command in ('update', 'revert'):
         pm = Progress('Syncing projects', 1)
       elif command == 'recurse':
         pm = Progress(' '.join(args), 1)
     work_queue = gclient_utils.ExecutionQueue(self._options.jobs, pm)
     for s in self.dependencies:
       work_queue.enqueue(s)
-    work_queue.flush(revision_overrides, command, args, options=self._options)
+    work_queue.flush(revision_overrides, command, args, options=self._options,
+                     ignore_requirements=ignore_requirements)
 
     # Once all the dependencies have been processed, it's now safe to run the
     # hooks.
     if not self._options.nohooks:
       self.RunHooksRecursively(self._options)
 
     if command == 'update':
       # Notify the user if there is an orphaned entry in their working copy.
       # Only delete the directory if there are no changes in it, and
       # delete_unversioned_trees is set to true.
@@ skipping 139 matching lines @@
 
 @attr('usage', '[command] [args ...]')
 def CMDrecurse(parser, args):
   """Operates on all the entries.
 
   Runs a shell command on all entries.
   """
   # Stop parsing at the first non-arg so that these go through to the command
   parser.disable_interspersed_args()
   parser.add_option('-s', '--scm', action='append', default=[],
-                    help='choose scm types to operate upon')
+                    help='Choose scm types to operate upon.')
   parser.add_option('-i', '--ignore', action='store_true',
-                    help='continue processing in case of non zero return code')
+                    help='Ignore non-zero return codes from subcommands.')
+  parser.add_option('--prepend-dir', action='store_true',
+                    help='Prepend relative dir for use with git <cmd> --null.')
   options, args = parser.parse_args(args)
   if not args:
     print >> sys.stderr, 'Need to supply a command!'
     return 1
   root_and_entries = gclient_utils.GetGClientRootAndEntries()
   if not root_and_entries:
     print >> sys.stderr, (
         'You need to run gclient sync at least once to use \'recurse\'.\n'
         'This is because .gclient_entries needs to exist and be up to date.')
     return 1
 
   # Normalize options.scm to a set()
   scm_set = set()
   for scm in options.scm:
     scm_set.update(scm.split(','))
   options.scm = scm_set
 
   options.nohooks = True
   client = GClient.LoadCurrentConfig(options)
-  return client.RunOnDeps('recurse', args)
+  return client.RunOnDeps('recurse', args, ignore_requirements=True)
 
 
 @attr('usage', '[args ...]')
 def CMDfetch(parser, args):
   """Fetches upstream commits for all modules.
 
 Completely git-specific. Simply runs 'git fetch [args ...]' for each module.
 """
   (options, args) = parser.parse_args(args)
-  args = ['-j%d' % options.jobs, '-s', 'git', 'git', 'fetch'] + args
-  return CMDrecurse(parser, args)
+  return CMDrecurse(Parser(), [
+      '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args)
+
+
+def CMDgrep(parser, args):
+  """Greps through git repos managed by gclient.
+
+Runs 'git grep [args...]' for each module.
+"""
+
+  # We can't use optparse because it will try to parse arguments sent
+  # to git grep and throw an error. :-(
+  if not args or re.match('(-h|--help)$', args[0]):
+    print >> sys.stderr, (
+        'Usage: gclient grep [-j <N>] git-grep-args...\n\n'
+        'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep '
+        '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to '
+        '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the'
+        ' end of your query.'
+        )
+    return 1
+
+  jobs_arg = ['--jobs=1']
+  if re.match(r'(-j|--jobs=)\d+$', args[0]):
+    jobs_arg, args = args[:1], args[1:]
+  elif re.match(r'(-j|--jobs)$', args[0]):
+    jobs_arg, args = args[:2], args[2:]
+
+  return CMDrecurse(
+      parser,
+      jobs_arg + ['--ignore', '--prepend-dir', '--scm=git', 'git', 'grep',
+                  '--null', '--color=Always'] + args)
 
 
 @attr('usage', '[url] [safesync url]')
 def CMDconfig(parser, args):
   """Create a .gclient file in the current directory.
 
 This specifies the configuration for further commands. After update/sync,
 top-level DEPS files in each module are read to determine dependent
 modules to operate on as well. If optional [url] parameter is
 provided, then configuration is read from a specified Subversion server
@@ skipping 397 matching lines @@
   except (gclient_utils.Error, subprocess2.CalledProcessError), e:
     print >> sys.stderr, 'Error: %s' % str(e)
     return 1
 
 
 if '__main__' == __name__:
   fix_encoding.fix_encoding()
   sys.exit(Main(sys.argv[1:]))
 
 # vim: ts=2:sw=2:tw=80:et: