Add git-number script to calculate generation numbers for commits.

git_common.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Monkeypatch IMapIterator so that Ctrl-C can kill everything properly.
+# Derived from https://gist.github.com/aljungberg/626518
+import multiprocessing.pool
+from multiprocessing.pool import IMapIterator

[M-A Ruel, 2013/10/21 17:56:44]

Can't this be done later?

[iannucci, 2013/10/22 07:28:22]

'this' ? Parallelizing, or monkey-patching IMapIterator?

I would rather do parallelization now because:
  1) It works
  2) It's fast (I can make numbers if you like, but it was quite substantial
last I checked (though previously it was doing many more tasks in parallel))
  3) It's (IMO) pretty low readability cost

The monkey patch is so that Ctrl-C works, because python can't be bothered to
have a working multiprocessing library :D

[M-A Ruel, 2013/10/24 13:23:03]

Monkey patching, but I don't mind. Leave it there.

+def wrapper(func):
+  def wrap(self, timeout=None):
+    return func(self, timeout=timeout or 1e100)
+  return wrap
+IMapIterator.next = wrapper(IMapIterator.next)
+IMapIterator.__next__ = IMapIterator.next
+
+import contextlib
+import functools
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import binascii

[M-A Ruel, 2013/10/21 17:56:44]

Order

[iannucci, 2013/10/22 07:28:22]

Done.

+
+hexlify = binascii.hexlify

[M-A Ruel, 2013/10/21 17:56:44]

It's not used once. Why?

[iannucci, 2013/10/22 07:28:22]

It's imported (in git_number). Used for dealing with git hashes. Same for
unhexlify.

[M-A Ruel, 2013/10/24 13:23:03]

Then import it where it's used.

+unhexlify = binascii.unhexlify

[M-A Ruel, 2013/10/21 17:56:44]

It's used once, why?

+pathlify = lambda s: '/'.join('%02x' % ord(b) for b in s)
+
+VERBOSE_LEVEL = 0
+

[M-A Ruel, 2013/10/21 17:56:44]

2 lines

[iannucci, 2013/10/22 07:28:22]

Done.

+class CalledProcessError(Exception):
+  def __init__(self, returncode, cmd, output=None, out_err=None):
+    super(CalledProcessError, self).__init__()
+    self.returncode = returncode
+    self.cmd = cmd
+    self.output = output
+    self.out_err = out_err

[agable, 2013/10/21 20:16:42]

output and out_err are never used.

[iannucci, 2013/10/22 07:28:22]

True. Done.

+
+  def __str__(self):
+    return (
+        'Command "%s" returned non-zero exit status %d' %
+        (self.cmd, self.returncode))
+
+
+def memoize_one(f):

[agable, 2013/10/21 20:16:42]

Is there anywhere else in depot_tools this could live, that's even more general
than git_common?

[iannucci, 2013/10/22 07:28:22]

Not sure...

+  """

[M-A Ruel, 2013/10/21 17:56:44]

"""Memoizes a single-argument pure function.

[iannucci, 2013/10/22 07:28:22]

Done.

+  Memoizes a single-argument pure function.
+
+  Values of None are not cached.
+
+  Adds a mutable attribute to the decorated function:
+    * cache (dict) - Maps arg to f(arg)
+  """
+  cache = {}
+
+  @functools.wraps(f)
+  def inner(arg):
+    ret = cache.get(arg)
+    if ret is None:
+      ret = f(arg)
+      if ret is not None:
+        cache[arg] = ret
+    return ret
+  inner.cache = cache
+  inner.default_enabled = False
+
+  return inner
+
+
+def initer(orig, orig_args):

[agable, 2013/10/21 20:16:42]

Define this inside ScopedPool?

[iannucci, 2013/10/22 07:28:22]

Can't. Multprocessing sux and this must be importable. I'll add a docstring tho.

+  signal.signal(signal.SIGINT, signal.SIG_IGN)
+  if orig:
+    orig(*orig_args)
+
+
+@contextlib.contextmanager
+def ScopedPool(*args, **kwargs):
+  if kwargs.pop('kind', None) == 'threads':
+    pool = multiprocessing.pool.ThreadPool(*args, **kwargs)
+  else:
+    orig, orig_args = kwargs.get('initializer'), kwargs.get('initargs', ())
+    kwargs['initializer'] = initer
+    kwargs['initargs'] = orig, orig_args
+    pool = multiprocessing.pool.Pool(*args, **kwargs)
+
+  try:
+    yield pool
+    pool.close()
+  except:
+    pool.terminate()
+    raise
+  finally:
+    pool.join()
+
+
+class StatusPrinter(object):

[agable, 2013/10/21 20:16:42]

This is more like a ProgressPrinter, since it increments and prints an integer
count.

[iannucci, 2013/10/22 07:28:22]

Done.

+  """Threaded single-stat status message printer."""
+  def __init__(self, fmt):
+    """
+    Create a StatusPrinter.

[agable, 2013/10/21 20:16:42]

Bump up a line.

[iannucci, 2013/10/22 07:28:22]

Done.

+
+    Call .start() to get it going, or use it as a context manager which produces

[agable, 2013/10/21 20:16:42]

Doesn't actually have a .start() method.

[iannucci, 2013/10/22 07:28:22]

well how about that... Done.

+    a simple 'increment' method:
+
+    with StatusPrinter('(%%d/%d)' % 1000) as inc:
+      for i in xrange(1000):
+        # do stuff
+        if i % 10 == 0:
+          inc(10)
+
+    Args:
+      fmt - String format with a single '%d' where the counter value should go.
+    """
+    self.fmt = fmt
+    self._count = 0
+    self._dead = False
+    self._dead_cond = threading.Condition()
+    self._thread = threading.Thread(target=self._run)
+
+  @staticmethod
+  def _emit(s):
+    if VERBOSE_LEVEL > 0:
+      sys.stderr.write('\r'+s)
+      sys.stderr.flush()
+
+  def _run(self):
+    with self._dead_cond:
+      while not self._dead:
+        self._emit(self.fmt % self._count)
+        self._dead_cond.wait(.5)
+      self._emit((self.fmt+'\n') % self._count)
+
+  def inc(self, amount=1):
+    self._count += amount
+
+  def __enter__(self):
+    self._thread.start()
+    return self.inc
+
+  def __exit__(self, _exc_type, _exc_value, _traceback):
+    self._dead = True
+    with self._dead_cond:
+      self._dead_cond.notifyAll()
+    self._thread.join()
+    del self._thread
+
+
+def parse_committish(*committish):

[agable, 2013/10/21 20:16:42]

Docstring. Unclear why a committish would have multiple lines (would expect it
to be one hexadecimal hash). Also, ditto comment above about just defining
unhexlify here.

[iannucci, 2013/10/22 07:28:22]

Added comments to *lify, added docstring and renamed this function to make more
sense.

+  try:
+    return map(unhexlify, git_hash(*committish).splitlines())
+  except CalledProcessError:
+    raise Exception('%r does not seem to be a valid commitish.' % committish)
+
+
+def check_output(*popenargs, **kwargs):

[agable, 2013/10/21 20:16:42]

Docstriiing.

[iannucci, 2013/10/22 07:28:22]

Done.

+  kwargs.setdefault('stdout', subprocess.PIPE)
+  kwargs.setdefault('stderr', subprocess.PIPE)
+  indata = kwargs.pop('indata', None)
+  if indata is not None:
+    kwargs['stdin'] = subprocess.PIPE
+  process = subprocess.Popen(*popenargs, **kwargs)
+  output, out_err = process.communicate(indata)

[agable, 2013/10/21 20:16:42]

I thought communicate misbehaved if you didn't have stdin=PIPE. Also, stdout and
stderr aren't reserved works, so you could just use those instead of output and
out_err.

[iannucci, 2013/10/22 07:28:22]

We do this if indata is not-None.
I prefer 'output' since 'stdout' is a stream, whereas 'output' is a string.
out_err is gone now.

+  retcode = process.poll()
+  if retcode:

[agable, 2013/10/21 20:16:42]

You need to handle the case where poll() returns None, saying the process hasn't
terminated.

[iannucci, 2013/10/22 07:28:22]

communicate() guarantees that the process is ended. I switched poll() to
returncode to be more explicit.

+    cmd = kwargs.get('args')
+    if cmd is None:
+      cmd = popenargs[0]
+    raise CalledProcessError(retcode, cmd, output=output, out_err=out_err)
+  return output
+
+
+GIT_EXE = 'git.bat' if sys.platform.startswith('win') else 'git'

[M-A Ruel, 2013/10/21 17:56:44]

Constants should be at the top.

[iannucci, 2013/10/22 07:28:22]

Yep. Done.

+
+def run_git(*cmd, **kwargs):

[agable, 2013/10/21 20:16:42]

Doooooooc strriiiiiiing.

[iannucci, 2013/10/22 07:28:22]

Done.

[M-A Ruel, 2013/10/24 13:23:03]

I don't think this particular function needed a docstring. It's more trivial and
transparent even if longer than let's say parse_committish(), which is more
opage.

+  cmd = (GIT_EXE,) + cmd
+  if VERBOSE_LEVEL > 1:
+    print 'running:', " ".join(repr(tok) for tok in cmd)
+  ret = check_output(cmd, **kwargs)
+  ret = (ret or '').strip()
+  return ret
+
+
+def git_hash(*reflike):

[agable, 2013/10/21 20:16:42]

Like seriously. All functions are worth good docstrings.

[iannucci, 2013/10/22 07:28:22]

really? this is a one-line function...

"""Returns the git hash after rev-parsing |reflike|"""

No.

[M-A Ruel, 2013/10/24 13:23:03]

I think it's more about *what* this call does. It converts a reference into a
hash. I really don't mind either way, I just think the function is ill named.

First it's a git_common module, naming the functions with a git_ prefix is
unneeded, I'd prefer the function to be named reference_to_commit_hash() and
then it's self-descriptive enough *at call site* that it wouldn't even need a
docstring.

But I don't mind.

[iannucci, 2013/10/25 00:52:41]

Hm... at the callsite:

  hashes = git_hash('HEAD', 'HEAD~100', "deadbeef")

I read it as:
  'hashes is the git_hash of "HEAD", "HEAD~100", and "deadbeef"'. 

Maybe that's just me?
+1 on taking the git prefix off of everything, but reference_to_commit_hash
seems too verbose to me. New naming scheme is go-style-ish (minus the Capital
functions).

+  return run_git('rev-parse', *reflike)
+
+
+def git_intern_f(f, kind='blob'):

[M-A Ruel, 2013/10/21 17:56:44]

This function is worth a docstring.

[iannucci, 2013/10/22 07:28:22]

Done.

+  ret = run_git('hash-object', '-t', kind, '-w', '--stdin', stdin=f)
+  f.close()
+  return ret
+
+
+def git_tree(treeish, recurse=False):
+  ret = {}
+  opts = ['ls-tree', '--full-tree']
+  if recurse:
+    opts += ['-r']
+  opts.append(treeish)
+  try:
+    for line in run_git(*opts).splitlines():
+      if not line:
+        continue
+      mode, typ, ref, name = line.split(None, 3)
+      ret[name] = (mode, typ, ref)
+  except CalledProcessError:
+    return None
+  return ret
+
+
+def git_mktree(treedict):
+  """
+  Args:
+    treedict - { name: (mode, type, ref) }
+  """
+  with tempfile.TemporaryFile() as f:
+    for name, (mode, typ, ref) in treedict.iteritems():
+      f.write('%s %s %s\t%s\0' % (mode, typ, ref, name))
+    f.seek(0)
+    return run_git('mktree', '-z', stdin=f)