Add a git cache for gclient sync operations.

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 import logging
 import os
 import posixpath
 import re
@@ skipping 134 matching lines @@
     if not command in commands:
       raise gclient_utils.Error('Unknown command %s' % command)
 
     if not command in dir(self):
       raise gclient_utils.Error('Command %s not implemented in %s wrapper' % (
           command, self.__class__.__name__))
 
     return getattr(self, command)(options, args, file_list)
 
 
+class GitFilter(object):
+  PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
+
+  def __init__(self, nag_timer, predicate=None):
+    """Create a new GitFilter.

[Michael Moss, 2013/07/02 20:47:52]

I tend to overdo code comments, but even I'm offended by this one. Then again,
we do have """Wrapper for Git""" for the GitWrapper class right below :/

[iannucci, 2013/07/02 22:10:07]

Yeah, the state of docstrings in this file is fairly lamentable :/ I'll put
something more reasonable here...

+
+    Args:

[Michael Moss, 2013/07/02 20:47:52]

When documenting args, we generally document them all (doesn't pylint complain
if you don't?).

[iannucci, 2013/07/02 22:10:07]

Oops! Done.

+      predicate (f(line)): An optional function which is invoked for every line.
+        The line will be skipped if the predicate is False.

[Michael Moss, 2013/07/02 20:47:52]

At first I read this as "if predicate == False". Probably should say "if the
predicate returns False."

[iannucci, 2013/07/02 22:10:07]

Done.

+    """
+    self.last_time = 0
+    self.nag_timer = nag_timer
+    self.predicate = predicate
+
+  def __call__(self, line):
+    # git uses an escape sequence to clear the line; elide it.
+    esc = line.find(unichr(033))
+    if esc > -1:
+      line = line[:esc]
+    if self.predicate and not self.predicate(line):
+      return
+    now = time.time()
+    match = self.PERCENT_RE.match(line)
+    if not match:
+      self.last_time = 0
+    if (now - self.last_time) >= (self.nag_timer / 2):
+      self.last_time = now
+      print line
+
+
 class GitWrapper(SCMWrapper):
   """Wrapper for Git"""
 
   def __init__(self, url=None, root_dir=None, relpath=None):
     """Removes 'git+' fake prefix from git URL."""
     if url.startswith('git+http://') or url.startswith('git+https://'):
       url = url[4:]
     SCMWrapper.__init__(self, url, root_dir, relpath)
 
   @staticmethod
@@ skipping 125 matching lines @@
     rev_str = ' at %s' % revision
     files = []
 
     printed_path = False
     verbose = []
     if options.verbose:
       print('\n_____ %s%s' % (self.relpath, rev_str))
       verbose = ['--verbose']
       printed_path = True
 
+    url = self._CreateOrUpdateCache(url, options)
+
     if revision.startswith('refs/heads/'):
       rev_type = "branch"
     elif revision.startswith('origin/'):
       # For compatability with old naming, translate 'origin' to 'refs/heads'
       revision = revision.replace('origin/', 'refs/heads/')
       rev_type = "branch"
     else:
       # hash is also a tag, only make a distinction at checkout
       rev_type = "hash"
 
@@ skipping 374 matching lines @@
             '#Initial_checkout' ) % rev)
 
     return sha1
 
   def FullUrlForRelativeUrl(self, url):
     # Strip from last '/'
     # Equivalent to unix basename
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
+  @staticmethod
+  def _CacheFolder(url, options):
+    """Transforms a url into the path to the corresponding git cache.
+
+    Ex. (assuming cache_dir == '/cache')
+      IN:  https://chromium.googlesource.com/chromium/src
+      OUT: /cache/chromium.googlesource.com-chromium-src.git
+    """
+    idx = url.find('://')
+    if idx != -1:
+      url = url[idx+3:]
+    # Replace - with -- to avoid ambiguity. / with - to flatten folder structure
+    url = url.replace('-', '--').replace('/', '-')
+    if not url.endswith('.git'):
+      url += '.git'
+    return os.path.join(options.cache_dir, url)
+
+  def _CreateOrUpdateCache(self, url, options):
+    """Make a new git mirror or update existing mirror for |url|, and return the
+    mirror URI to clone from.
+
+    If no cache-dir is specified, just return |url| unchanged.
+    """
+    if not options.cache_dir:
+      return url
+    folder = self._CacheFolder(url, options)
+    v = ['-v'] if options.verbose else []
+    filter_fn = lambda l: '[up to date]' not in l
+    with options.cache_locks[folder]:
+      gclient_utils.safe_makedirs(options.cache_dir)
+      if not os.path.exists(os.path.join(folder, 'config')):
+        gclient_utils.rmtree(folder)
+        self._Run(['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
+                                   '--progress', '--mirror', url, folder],
+                  options, git_filter=True, filter_fn=filter_fn,
+                  cwd=options.cache_dir)
+      else:
+        # Would normally use `git remote update`, but it doesn't support
+        # --progress, so use fetch instead.

[szager1, 2013/07/02 20:58:21]

existing_url = self._Run(['config', 'remote.origin.url'])
if existing_url != url:
  raise ...

[iannucci, 2013/07/02 22:10:07]

Why? The premise is that it doesn't matter what access method is used for a
given domain+path. This'll let us make the cache repos use whatever the best
access method/remotes we want on the bots later (i.e. for forming a mesh of git
mirrors in the golo, switching to a different auth scheme if necessary, etc.).

[szager1, 2013/07/02 22:14:05]

This is the collision-detection logic.  Maybe strip out the hostname and just
match the path; it's unlikely that we'll be switching paths.

+        self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
+                  options, git_filter=True, filter_fn=filter_fn, cwd=folder)
+    return folder
+
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
 
     Once we've cloned the repo, we checkout a working branch if the specified
     revision is a branch head. If it is a tag or a specific commit, then we
     leave HEAD detached as it makes future updates simpler -- in this case the
     user should first create a new branch or switch to an existing branch before
     making changes in the repo."""
     if not options.verbose:
       # git clone doesn't seem to insert a newline properly before printing
       # to stdout
       print('')
     clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--progress']
+    if options.cache_dir:
+      clone_cmd.append('--shared')
     if revision.startswith('refs/heads/'):
       clone_cmd.extend(['-b', revision.replace('refs/heads/', '')])
       detach_head = False
     else:
       detach_head = True
     if options.verbose:
       clone_cmd.append('--verbose')
     clone_cmd.extend([url, self.checkout_path])
 
     # If the parent directory does not exist, Git clone on Windows will not
     # create it, so we need to do it manually.
     parent_dir = os.path.dirname(self.checkout_path)
     if not os.path.exists(parent_dir):
       gclient_utils.safe_makedirs(parent_dir)
 
-    percent_re = re.compile('.* ([0-9]{1,2})% .*')
-    def _GitFilter(line):
-      # git uses an escape sequence to clear the line; elide it.
-      esc = line.find(unichr(033))
-      if esc > -1:
-        line = line[:esc]
-      match = percent_re.match(line)
-      if not match or not int(match.group(1)) % 10:
-        print '%s' % line
-
     for _ in range(3):
       try:
-        self._Run(clone_cmd, options, cwd=self._root_dir, filter_fn=_GitFilter,
-                  print_stdout=False)
+        self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
         break
       except subprocess2.CalledProcessError, e:
         # Too bad we don't have access to the actual output yet.
         # We should check for "transfer closed with NNN bytes remaining to
         # read". In the meantime, just make sure .git exists.
         if (e.returncode == 128 and
             os.path.exists(os.path.join(self.checkout_path, '.git'))):
           print(str(e))
           print('Retrying...')
           continue
@@ skipping 194 matching lines @@
             if options.verbose:
               fetch_cmd.append('--verbose')
             self._Run(fetch_cmd, options)
           break
         except subprocess2.CalledProcessError, e:
           print(str(e))
           print('Retrying in %.1f seconds...' % backoff_time)
           time.sleep(backoff_time)
           backoff_time *= 1.3
 
-  def _Run(self, args, options, **kwargs):
+  def _Run(self, args, _options, git_filter=False, **kwargs):
     kwargs.setdefault('cwd', self.checkout_path)
-    kwargs.setdefault('print_stdout', True)
     kwargs.setdefault('nag_timer', self.nag_timer)
     kwargs.setdefault('nag_max', self.nag_max)
+    if git_filter:
+      kwargs['filter_fn'] = GitFilter(kwargs['nag_timer'],
+                                      kwargs.get('filter_fn'))
+      kwargs.setdefault('print_stdout', False)
+    else:
+      kwargs.setdefault('print_stdout', True)
     stdout = kwargs.get('stdout', sys.stdout)
     stdout.write('\n________ running \'git %s\' in \'%s\'\n' % (
                  ' '.join(args), kwargs['cwd']))
     gclient_utils.CheckCallAndFilter(['git'] + args, **kwargs)
 
 
 class SVNWrapper(SCMWrapper):
   """ Wrapper for SVN """
 
   @staticmethod
@@ skipping 375 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command