Add a git cache for gclient sync operations.

gclient_scm.py

Index: gclient_scm.py
diff --git a/gclient_scm.py b/gclient_scm.py
index 5c3929a9639792e358a350e311b65a152664735e..f4c09d6dd6ee78cb9451e6eff6c71d40331fbaa7 100644
--- a/gclient_scm.py
+++ b/gclient_scm.py
@@ -152,6 +152,36 @@ class SCMWrapper(object):
     return getattr(self, command)(options, args, file_list)
 
 
+class GitFilter(object):
+  PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
+
+  def __init__(self, nag_timer, predicate=None):
+    """Create a new GitFilter.

[Michael Moss, 2013/07/02 20:47:52]

I tend to overdo code comments, but even I'm offended by this one. Then again,
we do have """Wrapper for Git""" for the GitWrapper class right below :/

[iannucci, 2013/07/02 22:10:07]

Yeah, the state of docstrings in this file is fairly lamentable :/ I'll put
something more reasonable here...

+
+    Args:

[Michael Moss, 2013/07/02 20:47:52]

When documenting args, we generally document them all (doesn't pylint complain
if you don't?).

[iannucci, 2013/07/02 22:10:07]

Oops! Done.

+      predicate (f(line)): An optional function which is invoked for every line.
+        The line will be skipped if the predicate is False.

[Michael Moss, 2013/07/02 20:47:52]

At first I read this as "if predicate == False". Probably should say "if the
predicate returns False."

[iannucci, 2013/07/02 22:10:07]

Done.

+    """
+    self.last_time = 0
+    self.nag_timer = nag_timer
+    self.predicate = predicate
+
+  def __call__(self, line):
+    # git uses an escape sequence to clear the line; elide it.
+    esc = line.find(unichr(033))
+    if esc > -1:
+      line = line[:esc]
+    if self.predicate and not self.predicate(line):
+      return
+    now = time.time()
+    match = self.PERCENT_RE.match(line)
+    if not match:
+      self.last_time = 0
+    if (now - self.last_time) >= (self.nag_timer / 2):
+      self.last_time = now
+      print line
+
+
 class GitWrapper(SCMWrapper):
   """Wrapper for Git"""
 
@@ -297,6 +327,8 @@ class GitWrapper(SCMWrapper):
       verbose = ['--verbose']
       printed_path = True
 
+    url = self._CreateOrUpdateCache(url, options)
+
     if revision.startswith('refs/heads/'):
       rev_type = "branch"
     elif revision.startswith('origin/'):
@@ -691,6 +723,49 @@ class GitWrapper(SCMWrapper):
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
+  @staticmethod
+  def _CacheFolder(url, options):
+    """Transforms a url into the path to the corresponding git cache.
+
+    Ex. (assuming cache_dir == '/cache')
+      IN:  https://chromium.googlesource.com/chromium/src
+      OUT: /cache/chromium.googlesource.com-chromium-src.git
+    """
+    idx = url.find('://')
+    if idx != -1:
+      url = url[idx+3:]
+    # Replace - with -- to avoid ambiguity. / with - to flatten folder structure
+    url = url.replace('-', '--').replace('/', '-')
+    if not url.endswith('.git'):
+      url += '.git'
+    return os.path.join(options.cache_dir, url)
+
+  def _CreateOrUpdateCache(self, url, options):
+    """Make a new git mirror or update existing mirror for |url|, and return the
+    mirror URI to clone from.
+
+    If no cache-dir is specified, just return |url| unchanged.
+    """
+    if not options.cache_dir:
+      return url
+    folder = self._CacheFolder(url, options)
+    v = ['-v'] if options.verbose else []
+    filter_fn = lambda l: '[up to date]' not in l
+    with options.cache_locks[folder]:
+      gclient_utils.safe_makedirs(options.cache_dir)
+      if not os.path.exists(os.path.join(folder, 'config')):
+        gclient_utils.rmtree(folder)
+        self._Run(['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
+                                   '--progress', '--mirror', url, folder],
+                  options, git_filter=True, filter_fn=filter_fn,
+                  cwd=options.cache_dir)
+      else:
+        # Would normally use `git remote update`, but it doesn't support
+        # --progress, so use fetch instead.

[szager1, 2013/07/02 20:58:21]

existing_url = self._Run(['config', 'remote.origin.url'])
if existing_url != url:
  raise ...

[iannucci, 2013/07/02 22:10:07]

Why? The premise is that it doesn't matter what access method is used for a
given domain+path. This'll let us make the cache repos use whatever the best
access method/remotes we want on the bots later (i.e. for forming a mesh of git
mirrors in the golo, switching to a different auth scheme if necessary, etc.).

[szager1, 2013/07/02 22:14:05]

This is the collision-detection logic.  Maybe strip out the hostname and just
match the path; it's unlikely that we'll be switching paths.

+        self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
+                  options, git_filter=True, filter_fn=filter_fn, cwd=folder)
+    return folder
+
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
 
@@ -704,6 +779,8 @@ class GitWrapper(SCMWrapper):
       # to stdout
       print('')
     clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--progress']
+    if options.cache_dir:
+      clone_cmd.append('--shared')
     if revision.startswith('refs/heads/'):
       clone_cmd.extend(['-b', revision.replace('refs/heads/', '')])
       detach_head = False
@@ -719,20 +796,9 @@ class GitWrapper(SCMWrapper):
     if not os.path.exists(parent_dir):
       gclient_utils.safe_makedirs(parent_dir)
 
-    percent_re = re.compile('.* ([0-9]{1,2})% .*')
-    def _GitFilter(line):
-      # git uses an escape sequence to clear the line; elide it.
-      esc = line.find(unichr(033))
-      if esc > -1:
-        line = line[:esc]
-      match = percent_re.match(line)
-      if not match or not int(match.group(1)) % 10:
-        print '%s' % line
-
     for _ in range(3):
       try:
-        self._Run(clone_cmd, options, cwd=self._root_dir, filter_fn=_GitFilter,
-                  print_stdout=False)
+        self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
         break
       except subprocess2.CalledProcessError, e:
         # Too bad we don't have access to the actual output yet.
@@ -947,11 +1013,16 @@ class GitWrapper(SCMWrapper):
           time.sleep(backoff_time)
           backoff_time *= 1.3
 
-  def _Run(self, args, options, **kwargs):
+  def _Run(self, args, _options, git_filter=False, **kwargs):
     kwargs.setdefault('cwd', self.checkout_path)
-    kwargs.setdefault('print_stdout', True)
     kwargs.setdefault('nag_timer', self.nag_timer)
     kwargs.setdefault('nag_max', self.nag_max)
+    if git_filter:
+      kwargs['filter_fn'] = GitFilter(kwargs['nag_timer'],
+                                      kwargs.get('filter_fn'))
+      kwargs.setdefault('print_stdout', False)
+    else:
+      kwargs.setdefault('print_stdout', True)
     stdout = kwargs.get('stdout', sys.stdout)
     stdout.write('\n________ running \'git %s\' in \'%s\'\n' % (
                  ' '.join(args), kwargs['cwd']))