Add a git cache for gclient sync operations.

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
+import collections
 import logging
 import os
 import posixpath
 import re
 import sys
+import threading
 import time
 
 import gclient_utils
 import scm
 import subprocess2
 
 
 THIS_FILE_PATH = os.path.abspath(__file__)
 
 
@@ skipping 94 matching lines @@
 # SCMWrapper base class
 
 class SCMWrapper(object):
   """Add necessary glue between all the supported SCM.
 
   This is the abstraction layer to bind to different SCM.
   """
   nag_timer = 30
   nag_max = 6
 
+  cache_dir = None

[Michael Moss, 2013/07/03 14:34:23]

Since this is git-only, any reason not to put it in GitWrapper (and update
references in gclient.py)?

[szager1, 2013/07/03 16:49:52]

Yeah, I agree with mmoss.

[iannucci, 2013/07/03 19:07:32]

... What a good idea!

And THAT's why you Always get code reviews...

+  # If a given cache is used in a solution more than once, prevent multiple
+  # threads from updating it simultaneously.
+  cache_locks = collections.defaultdict(threading.Lock)
+
   def __init__(self, url=None, root_dir=None, relpath=None):
     self.url = url
     self._root_dir = root_dir
     if self._root_dir:
       self._root_dir = self._root_dir.replace('/', os.sep)
     self.relpath = relpath
     if self.relpath:
       self.relpath = self.relpath.replace('/', os.sep)
     if self.relpath and self._root_dir:
       self.checkout_path = os.path.join(self._root_dir, self.relpath)
 
   def RunCommand(self, command, options, args, file_list=None):
     # file_list will have all files that are modified appended to it.
     if file_list is None:
       file_list = []
 
     commands = ['cleanup', 'update', 'updatesingle', 'revert',
                 'revinfo', 'status', 'diff', 'pack', 'runhooks']
 
     if not command in commands:
       raise gclient_utils.Error('Unknown command %s' % command)
 
     if not command in dir(self):
       raise gclient_utils.Error('Command %s not implemented in %s wrapper' % (
           command, self.__class__.__name__))
 
     return getattr(self, command)(options, args, file_list)
 
 
+class GitFilter(object):
+  """A filter_fn implementation for quieting down git output messages.
+
+  Allows a custom function to skip certain lines (predicate), and will throttle
+  the output of percentage completed lines to only output every X seconds.
+  """
+  PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
+
+  def __init__(self, time_throttle=0, predicate=None):
+    """
+    Args:
+      time_throttle (int): GitFilter will throttle 'noisy' output (such as the
+        XX% complete messages) to only be printed at least |time_throttle|
+        seconds apart.
+      predicate (f(line)): An optional function which is invoked for every line.
+        The line will be skipped if predicate(line) returns False.
+    """
+    self.last_time = 0
+    self.time_throttle = time_throttle
+    self.predicate = predicate
+
+  def __call__(self, line):
+    # git uses an escape sequence to clear the line; elide it.
+    esc = line.find(unichr(033))
+    if esc > -1:
+      line = line[:esc]
+    if self.predicate and not self.predicate(line):
+      return
+    now = time.time()
+    match = self.PERCENT_RE.match(line)
+    if not match:
+      self.last_time = 0
+    if (now - self.last_time) >= self.time_throttle:
+      self.last_time = now
+      print line
+
+
 class GitWrapper(SCMWrapper):
   """Wrapper for Git"""
 
   def __init__(self, url=None, root_dir=None, relpath=None):
     """Removes 'git+' fake prefix from git URL."""
     if url.startswith('git+http://') or url.startswith('git+https://'):
       url = url[4:]
     SCMWrapper.__init__(self, url, root_dir, relpath)
 
   @staticmethod
@@ skipping 125 matching lines @@
     rev_str = ' at %s' % revision
     files = []
 
     printed_path = False
     verbose = []
     if options.verbose:
       print('\n_____ %s%s' % (self.relpath, rev_str))
       verbose = ['--verbose']
       printed_path = True
 
+    url = self._CreateOrUpdateCache(url, options)
+
     if revision.startswith('refs/heads/'):
       rev_type = "branch"
     elif revision.startswith('origin/'):
       # For compatability with old naming, translate 'origin' to 'refs/heads'
       revision = revision.replace('origin/', 'refs/heads/')
       rev_type = "branch"
     else:
       # hash is also a tag, only make a distinction at checkout
       rev_type = "hash"
 
@@ skipping 374 matching lines @@
             '#Initial_checkout' ) % rev)
 
     return sha1
 
   def FullUrlForRelativeUrl(self, url):
     # Strip from last '/'
     # Equivalent to unix basename
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
+  @staticmethod
+  def _NormalizeGitURL(url):
+    '''Takes a git url, strips the scheme, and ensures it ends with '.git'.'''
+    idx = url.find('://')
+    if idx != -1:
+      url = url[idx+3:]
+    if not url.endswith('.git'):
+      url += '.git'
+    return url
+
+  def _CreateOrUpdateCache(self, url, options):
+    """Make a new git mirror or update existing mirror for |url|, and return the
+    mirror URI to clone from.
+
+    If no cache-dir is specified, just return |url| unchanged.
+    """
+    if not self.cache_dir:
+      return url
+
+    # Replace - with -- to avoid ambiguity. / with - to flatten folder structure
+    folder = os.path.join(
+      self.cache_dir,
+      self._NormalizeGitURL(url).replace('-', '--').replace('/', '-'))
+
+    v = ['-v'] if options.verbose else []
+    filter_fn = lambda l: '[up to date]' not in l
+    with self.cache_locks[folder]:
+      gclient_utils.safe_makedirs(self.cache_dir)
+      if not os.path.exists(os.path.join(folder, 'config')):
+        gclient_utils.rmtree(folder)
+        self._Run(['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
+                                   '--progress', '--mirror', url, folder],
+                  options, git_filter=True, filter_fn=filter_fn,
+                  cwd=self.cache_dir)
+      else:
+        # For now, assert that host/path/to/repo.git is identical. We may want
+        # to relax this restriction in the future to allow for smarter cache
+        # repo update schemes (such as pulling the same repo, but from a
+        # different host).
+        existing_url = self._Capture(['config', 'remote.origin.url'],
+                                     cwd=folder)
+        assert self._NormalizeGitURL(existing_url) == self._NormalizeGitURL(url)
+
+        # Would normally use `git remote update`, but it doesn't support
+        # --progress, so use fetch instead.
+        self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
+                  options, git_filter=True, filter_fn=filter_fn, cwd=folder)
+    return folder
+
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
 
     Once we've cloned the repo, we checkout a working branch if the specified
     revision is a branch head. If it is a tag or a specific commit, then we
     leave HEAD detached as it makes future updates simpler -- in this case the
     user should first create a new branch or switch to an existing branch before
     making changes in the repo."""
     if not options.verbose:
       # git clone doesn't seem to insert a newline properly before printing
       # to stdout
       print('')
     clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--progress']
+    if self.cache_dir:
+      clone_cmd.append('--shared')
     if revision.startswith('refs/heads/'):
       clone_cmd.extend(['-b', revision.replace('refs/heads/', '')])
       detach_head = False
     else:
       detach_head = True
     if options.verbose:
       clone_cmd.append('--verbose')
     clone_cmd.extend([url, self.checkout_path])
 
     # If the parent directory does not exist, Git clone on Windows will not
     # create it, so we need to do it manually.
     parent_dir = os.path.dirname(self.checkout_path)
     if not os.path.exists(parent_dir):
       gclient_utils.safe_makedirs(parent_dir)
 
-    percent_re = re.compile('.* ([0-9]{1,2})% .*')
-    def _GitFilter(line):
-      # git uses an escape sequence to clear the line; elide it.
-      esc = line.find(unichr(033))
-      if esc > -1:
-        line = line[:esc]
-      match = percent_re.match(line)
-      if not match or not int(match.group(1)) % 10:
-        print '%s' % line
-
     for _ in range(3):
       try:
-        self._Run(clone_cmd, options, cwd=self._root_dir, filter_fn=_GitFilter,
-                  print_stdout=False)
+        self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
         break
       except subprocess2.CalledProcessError, e:
         # Too bad we don't have access to the actual output yet.
         # We should check for "transfer closed with NNN bytes remaining to
         # read". In the meantime, just make sure .git exists.
         if (e.returncode == 128 and
             os.path.exists(os.path.join(self.checkout_path, '.git'))):
           print(str(e))
           print('Retrying...')
           continue
@@ skipping 164 matching lines @@
       print('\n_____ found an unreferenced commit and saved it as \'%s\'' %
           name)
 
   def _GetCurrentBranch(self):
     # Returns name of current branch or None for detached HEAD
     branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD'])
     if branch == 'HEAD':
       return None
     return branch
 
-  def _Capture(self, args):
+  def _Capture(self, args, cwd=None):
     return subprocess2.check_output(
         ['git'] + args,
         stderr=subprocess2.VOID,
         nag_timer=self.nag_timer,
         nag_max=self.nag_max,
-        cwd=self.checkout_path).strip()
+        cwd=cwd or self.checkout_path).strip()
 
   def _UpdateBranchHeads(self, options, fetch=False):
     """Adds, and optionally fetches, "branch-heads" refspecs if requested."""
     if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
       backoff_time = 5
       for _ in range(3):
         try:
           config_cmd = ['config', 'remote.origin.fetch',
                         '+refs/branch-heads/*:refs/remotes/branch-heads/*',
                         '^\\+refs/branch-heads/\\*:.*$']
           self._Run(config_cmd, options)
           if fetch:
             fetch_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'fetch', 'origin']
             if options.verbose:
               fetch_cmd.append('--verbose')
             self._Run(fetch_cmd, options)
           break
         except subprocess2.CalledProcessError, e:
           print(str(e))
           print('Retrying in %.1f seconds...' % backoff_time)
           time.sleep(backoff_time)
           backoff_time *= 1.3
 
-  def _Run(self, args, options, **kwargs):
+  def _Run(self, args, _options, git_filter=False, **kwargs):
     kwargs.setdefault('cwd', self.checkout_path)
-    kwargs.setdefault('print_stdout', True)
     kwargs.setdefault('nag_timer', self.nag_timer)
     kwargs.setdefault('nag_max', self.nag_max)
+    if git_filter:
+      kwargs['filter_fn'] = GitFilter(kwargs['nag_timer'] / 2,
+                                      kwargs.get('filter_fn'))
+      kwargs.setdefault('print_stdout', False)
+    else:
+      kwargs.setdefault('print_stdout', True)
     stdout = kwargs.get('stdout', sys.stdout)
     stdout.write('\n________ running \'git %s\' in \'%s\'\n' % (
                  ' '.join(args), kwargs['cwd']))
     gclient_utils.CheckCallAndFilter(['git'] + args, **kwargs)
 
 
 class SVNWrapper(SCMWrapper):
   """ Wrapper for SVN """
 
   @staticmethod
@@ skipping 375 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command