Add color support to git cl and fetch issue properties in parallel.

git_cl.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Copyright (C) 2008 Evan Martin <martine@danga.com>
 
 """A git-command for integrating reviews on Rietveld."""
 
 import difflib
 from distutils.version import LooseVersion
 import json
 import logging
 import optparse
 import os
+import Queue
 import re
 import stat
 import sys
 import textwrap
+import threading
 import urllib2
 import urlparse
 
 try:
   import readline  # pylint: disable=F0401,W0611
 except ImportError:
   pass
 
 
 from third_party import colorama
@@ skipping 375 matching lines @@
     self.LazyUpdateIfNeeded()
     return RunGit(['config', param], **kwargs).strip()
 
 
 def ShortBranchName(branch):
   """Convert a name like 'refs/heads/foo' to just 'foo'."""
   return branch.replace('refs/heads/', '')
 
 
 class Changelist(object):
-  def __init__(self, branchref=None):
+  def __init__(self, branchref=None, issue=None):
     # Poke settings so we get the "configure your server" message if necessary.
     global settings
     if not settings:
       # Happens when git_cl.py is used as a utility library.
       settings = Settings()
     settings.GetDefaultServerUrl()
     self.branchref = branchref
     if self.branchref:
       self.branch = ShortBranchName(self.branchref)
     else:
       self.branch = None
     self.rietveld_server = None
     self.upstream_branch = None
-    self.has_issue = False
-    self.issue = None
+    self.lookedup_issue = False
+    self.issue = issue or None
     self.has_description = False
     self.description = None
-    self.has_patchset = False
+    self.lookedup_patchset = False
     self.patchset = None
     self._rpc_server = None
     self.cc = None
     self.watchers = ()
     self._remote = None
+    self._props = None
 
   def GetCCList(self):
     """Return the users cc'd on this CL.
 
     Return is a string suitable for passing to gcl with the --cc flag.
     """
     if self.cc is None:
       base_cc = settings  .GetDefaultCCList()
       more_cc = ','.join(self.watchers)
       self.cc = ','.join(filter(None, (base_cc, more_cc))) or ''
@@ skipping 145 matching lines @@
   def GetRemoteUrl(self):
     """Return the configured remote URL, e.g. 'git://example.org/foo.git/'.
 
     Returns None if there is no remote.
     """
     remote, _ = self.GetRemoteBranch()
     return RunGit(['config', 'remote.%s.url' % remote], error_ok=True).strip()
 
   def GetIssue(self):
     """Returns the issue number as a int or None if not set."""
-    if not self.has_issue:
+    if self.issue is None and not self.lookedup_issue:
       issue = RunGit(['config', self._IssueSetting()], error_ok=True).strip()
-      if issue:
-        self.issue = int(issue)
-      else:
-        self.issue = None
-      self.has_issue = True
+      self.issue = int(issue) or None if issue else None
+      self.lookedup_issue = True
     return self.issue
 
   def GetRietveldServer(self):
     if not self.rietveld_server:
       # If we're on a branch then get the server potentially associated
       # with that branch.
       if self.GetIssue():
         self.rietveld_server = gclient_utils.UpgradeToHttps(RunGit(
             ['config', self._RietveldServer()], error_ok=True).strip())
       if not self.rietveld_server:
@@ skipping 28 matching lines @@
                 '\nFailed to fetch issue description. HTTP error ' + e.code)
       self.has_description = True
     if pretty:
       wrapper = textwrap.TextWrapper()
       wrapper.initial_indent = wrapper.subsequent_indent = '  '
       return wrapper.fill(self.description)
     return self.description
 
   def GetPatchset(self):
     """Returns the patchset number as a int or None if not set."""
-    if not self.has_patchset:
+    if self.patchset is None and not self.lookedup_patchset:
       patchset = RunGit(['config', self._PatchsetSetting()],
                         error_ok=True).strip()
-      if patchset:
-        self.patchset = int(patchset)
-      else:
-        self.patchset = None
-      self.has_patchset = True
+      self.patchset = int(patchset) or None if patchset else None
+      self.lookedup_patchset = True
     return self.patchset
 
   def SetPatchset(self, patchset):
     """Set this branch's patchset.  If patchset=0, clears the patchset."""
     if patchset:
       RunGit(['config', self._PatchsetSetting(), str(patchset)])
+      self.patchset = patchset
     else:
       RunGit(['config', '--unset', self._PatchsetSetting()],
              stderr=subprocess2.PIPE, error_ok=True)
-    self.has_patchset = False
+      self.patchset = None
 
-  def GetMostRecentPatchset(self, issue):
-    return self.RpcServer().get_issue_properties(
-        int(issue), False)['patchsets'][-1]
+  def GetMostRecentPatchset(self):
+    return self.GetIssueProperties()['patchsets'][-1]
 
   def GetPatchSetDiff(self, issue, patchset):
     return self.RpcServer().get(
         '/download/issue%s_%s.diff' % (issue, patchset))
 
+  def GetIssueProperties(self):
+    if self._props is None:
+      issue = self.GetIssue()
+      if not issue:
+        self._props = {}
+      else:
+        self._props = self.RpcServer().get_issue_properties(issue, True)
+    return self._props
+
   def GetApprovingReviewers(self):
-    return get_approving_reviewers(
-        self.RpcServer().get_issue_properties(self.GetIssue(), True))
+    return get_approving_reviewers(self.GetIssueProperties())
 
   def SetIssue(self, issue):
     """Set this branch's issue.  If issue=0, clears the issue."""
     if issue:
+      self.issue = issue
       RunGit(['config', self._IssueSetting(), str(issue)])
       if self.rietveld_server:
         RunGit(['config', self._RietveldServer(), self.rietveld_server])
     else:
       RunGit(['config', '--unset', self._IssueSetting()])
-      self.SetPatchset(0)
-    self.has_issue = False
+      self.issue = None
+      self.SetPatchset(None)
 
   def GetChange(self, upstream_branch, author):
     if not self.GitSanityChecks(upstream_branch):
       DieWithError('\nGit sanity check failure')
 
     env = os.environ.copy()
     # 'cat' is a magical git string that disables pagers on all platforms.
     env['GIT_PAGER'] = 'cat'
 
     root = RunCommand(['git', 'rev-parse', '--show-cdup'], env=env).strip()
@@ skipping 347 matching lines @@
   else:
     print("Setting base-url to %s" % args[0])
     return RunGit(['config', 'branch.%s.base-url' % branch, args[0]],
                   error_ok=False).strip()
 
 
 def CMDstatus(parser, args):
   """show status of changelists"""
   parser.add_option('--field',
                     help='print only specific field (desc|id|patch|url)')
+  parser.add_option('-f', '--fast', action='store_true',
+                    help='Do not retrieve review status')
   (options, args) = parser.parse_args(args)
 
   if options.field:
     cl = Changelist()
     if options.field.startswith('desc'):
       print cl.GetDescription()
     elif options.field == 'id':
       issueid = cl.GetIssue()
       if issueid:
         print issueid
     elif options.field == 'patch':
       patchset = cl.GetPatchset()
       if patchset:
         print patchset
     elif options.field == 'url':
       url = cl.GetIssueURL()
       if url:
         print url
     return 0
 
   branches = RunGit(['for-each-ref', '--format=%(refname)', 'refs/heads'])
   if not branches:
     print('No local branch found.')
     return 0
 
   changes = (Changelist(branchref=b) for b in branches.splitlines())
   branches = dict((c.GetBranch(), c.GetIssueURL()) for c in changes)
   alignment = max(5, max(len(b) for b in branches))
   print 'Branches associated with reviews:'
+  # Adhoc thread pool to request data concurrently.
+  output = Queue.Queue()
+
+  # Silence upload.py otherwise it becomes unweldly.
+  upload.verbosity = 0
+
+  if not options.fast:
+    def fetch(b):
+      c = Changelist(branchref=b)
+      i = c.GetIssueURL()
+      try:
+        props = c.GetIssueProperties()
+        r = c.GetApprovingReviewers() if i else None
+        if not props.get('messages'):
+          r = None
+      except urllib2.HTTPError:
+        # The issue probably doesn't exist anymore.
+        i += ' (broken)'
+        r = None
+      output.put((b, i, r))
+
+    threads = [threading.Thread(target=fetch, args=(b,)) for b in branches]
+    for t in threads:
+      t.daemon = True
+      t.start()
+  else:
+    # Do not use GetApprovingReviewers(), since it requires an HTTP request.
+    for b in branches:
+      c = Changelist(branchref=b)
+      output.put((b, c.GetIssue(), None))
+
+  tmp = {}
+  alignment = max(5, max(len(ShortBranchName(b)) for b in branches))
   for branch in sorted(branches):
-    print "  %*s: %s" % (alignment, branch, branches[branch])
+    while branch not in tmp:
+      b, i, r = output.get()
+      tmp[b] = (i, r)
+    issue, reviewers = tmp.pop(branch)
+    if not issue:
+      color = Fore.WHITE
+    elif reviewers:
+      # Was approved.
+      color = Fore.GREEN
+    elif reviewers is None:
+      # No message was sent.
+      color = Fore.RED
+    else:
+      color = Fore.BLUE
+    print '  %*s: %s%s%s' % (
+          alignment, ShortBranchName(branch), color, issue, Fore.RESET)
+
   cl = Changelist()
   print
   print 'Current branch:',
   if not cl.GetIssue():
     print 'no issue assigned.'
     return 0
   print cl.GetBranch()
   print 'Issue number: %s (%s)' % (cl.GetIssue(), cl.GetIssueURL())
   print 'Issue description:'
   print cl.GetDescription(pretty=True)
@@ skipping 21 matching lines @@
 
 
 def CMDcomments(parser, args):
   """show review comments of the current changelist"""
   (_, args) = parser.parse_args(args)
   if args:
     parser.error('Unsupported argument: %s' % args)
 
   cl = Changelist()
   if cl.GetIssue():
-    data = cl.RpcServer().get_issue_properties(cl.GetIssue(), True)
+    data = cl.GetIssueProperties()
     for message in sorted(data['messages'], key=lambda x: x['date']):
       if message['disapproval']:
         color = Fore.RED
       elif message['approval']:
         color = Fore.GREEN
       elif message['sender'] == data['owner_email']:
         color = Fore.MAGENTA
       else:
         color = Fore.BLUE
       print '\n%s%s  %s%s' % (
@@ skipping 277 matching lines @@
     hook_results = cl.RunHook(committing=False,
                               may_prompt=not options.force,
                               verbose=options.verbose,
                               change=change)
     if not hook_results.should_continue():
       return 1
     if not options.reviewers and hook_results.reviewers:
       options.reviewers = hook_results.reviewers.split(',')
 
   if cl.GetIssue():
-    latest_patchset = cl.GetMostRecentPatchset(cl.GetIssue())
+    latest_patchset = cl.GetMostRecentPatchset()
     local_patchset = cl.GetPatchset()
     if latest_patchset and local_patchset and local_patchset != latest_patchset:
       print ('The last upload made from this repository was patchset #%d but '
             'the most recent patchset on the server is #%d.'
             % (local_patchset, latest_patchset))
       print ('Uploading will still work, but if you\'ve uploaded to this issue '
             'from another machine or branch the patch you\'re uploading now '
             'might not include those changes.')
       ask_for_data('About to upload; enter to confirm.')
 
@@ skipping 218 matching lines @@
       return 1
     viewvc_url = settings.GetViewVCUrl()
     if viewvc_url and revision:
       change_desc.append_footer('Committed: ' + viewvc_url + revision)
     elif revision:
       change_desc.append_footer('Committed: ' + revision)
     print ('Closing issue '
            '(you may be prompted for your codereview password)...')
     cl.UpdateDescription(change_desc.description)
     cl.CloseIssue()
-    props = cl.RpcServer().get_issue_properties(cl.GetIssue(), False)
+    props = cl.GetIssueProperties()
     patch_num = len(props['patchsets'])
     comment = "Committed patchset #%d manually as r%s" % (patch_num, revision)
     comment += ' (presubmit successful).' if not options.bypass_hooks else '.'
     cl.RpcServer().add_comment(cl.GetIssue(), comment)
-    cl.SetIssue(0)
+    cl.SetIssue(None)

[iannucci, 2013/07/23 21:19:25]

if you git cl issue 0, will that still do the right thing?

[M-A Ruel, 2013/07/23 23:22:29]

Yes, tested.

 
   if retcode == 0:
     hook = POSTUPSTREAM_HOOK_PATTERN % cmd
     if os.path.isfile(hook):
       RunCommand([hook, base_branch], error_ok=True)
 
   return 0
 
 
 @usage('[upstream branch to apply against]')
@@ skipping 38 matching lines @@
   if len(args) != 1:
     parser.print_help()
     return 1
   issue_arg = args[0]
 
   # TODO(maruel): Use apply_issue.py
   # TODO(ukai): use gerrit-cherry-pick for gerrit repository?
 
   if issue_arg.isdigit():
     # Input is an issue id.  Figure out the URL.
-    cl = Changelist()
     issue = int(issue_arg)
-    patchset = cl.GetMostRecentPatchset(issue)
+    cl = Changelist(issue)
+    patchset = cl.GetMostRecentPatchset()
     patch_data = cl.GetPatchSetDiff(issue, patchset)
   else:
     # Assume it's a URL to the patch. Default to https.
     issue_url = gclient_utils.UpgradeToHttps(issue_arg)
     match = re.match(r'.*?/issue(\d+)_(\d+).diff', issue_url)
     if not match:
       DieWithError('Must pass an issue ID or full URL for '
           '\'Download raw patch set\'')
     issue = int(match.group(1))
     patchset = int(match.group(2))
@@ skipping 186 matching lines @@
   if any('triggered' in b for b in builders_and_tests):
     print >> sys.stderr, (
         'ERROR You are trying to send a job to a triggered bot.  This type of'
         ' bot requires an\ninitial job from a parent (usually a builder).  '
         'Instead send your job to the parent.\n'
         'Bot list: %s' % builders_and_tests)
     return 1
 
   patchset = cl.GetPatchset()
   if not cl.GetPatchset():
-    patchset = cl.GetMostRecentPatchset(cl.GetIssue())
+    patchset = cl.GetMostRecentPatchset()
 
   cl.RpcServer().trigger_try_jobs(
       cl.GetIssue(), patchset, options.name, options.clobber, options.revision,
       builders_and_tests)
   print('Tried jobs on:')
   length = max(len(builder) for builder in builders_and_tests)
   for builder in sorted(builders_and_tests):
     print '  %*s: %s' % (length, builder, ','.join(builders_and_tests[builder]))
   return 0
 
@@ skipping 217 matching lines @@
   GenUsage(parser, 'help')
   return CMDhelp(parser, argv)
 
 
 if __name__ == '__main__':
   # These affect sys.stdout so do it outside of main() to simplify mocks in
   # unit testing.
   fix_encoding.fix_encoding()
   colorama.init()
   sys.exit(main(sys.argv[1:]))