Add WebKit bugzilla/git search to my_activity.py

my_activity.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Get stats about your activity.
 
 Example:
   - my_activity.py  for stats for the current week (last week on mondays).
   - my_activity.py -Q  for stats for last quarter.
   - my_activity.py -Y  for stats for this year.
   - my_activity.py -b 4/5/12  for stats since 4/5/12.
   - my_activity.py -b 4/5/12 -e 6/7/12 for stats between 4/5/12 and 6/7/12.
 """
 
 # These services typically only provide a created time and a last modified time
 # for each item for general queries. This is not enough to determine if there
 # was activity in a given time period. So, we first query for all things created
 # before end and modified after begin. Then, we get the details of each item and
 # check those details to determine if there was activity in the given period.
 # This means that query time scales mostly with (today() - begin).
 
 import cookielib
+import csv
 import datetime
 from datetime import datetime
 from datetime import timedelta
 from functools import partial
 import json
 import optparse
 import os
+import re
 import subprocess
 import sys
 import urllib
 import urllib2
 
 import rietveld
 from third_party import upload
 
+# Imported later, once options are set.
+webkitpy = None
+
 try:
   from dateutil.relativedelta import relativedelta # pylint: disable=F0401
 except ImportError:
   print 'python-dateutil package required'
   exit(1)
 
 # python-keyring provides easy access to the system keyring.
 try:
   import keyring  # pylint: disable=W0611,F0401
 except ImportError:
   print 'Consider installing python-keyring'
 
+def webkit_account(user):
+  if not webkitpy:
+    return None
+  committer_list = webkitpy.common.config.committers.CommitterList()
+  email = user + "@chromium.org"
+  return committer_list.account_by_email(email)
+
+def user_to_webkit_email(user):
+  account = webkit_account(user)
+  if not account:
+    return None
+  return account.emails[0]
+
+def user_to_webkit_owner_search(user):
+  account = webkit_account(user)
+  if not account:
+    return ['--author=%s@chromium.org' % user]
+  search = []
+  for email in account.emails:
+    search.append('--author=' + email)
+  # commit-bot is author for contributors who are not committers.
+  search.append('--grep=Patch by ' + account.full_name)
+  return search
+
+def user_to_webkit_reviewer_search(user):
+  committer_list = webkitpy.common.config.committers.CommitterList()
+  email = user + "@chromium.org"
+  account = committer_list.reviewer_by_email(email)
+  if not account:
+    return []
+  return ['--grep=Reviewed by ' + account.full_name]
 
 rietveld_instances = [
   {
     'url': 'codereview.chromium.org',
     'shorturl': 'crrev.com',
     'supports_owner_modified_query': True,
     'requires_auth': False,
     'email_domain': 'chromium.org',
   },
   {
@@ skipping 40 matching lines @@
     'name': 'chrome-os-partner',
   },
   {
     'name': 'google-breakpad',
   },
   {
     'name': 'gyp',
   }
 ]
 
+bugzilla_instances = [
+  {
+    'search_url': 'http://bugs.webkit.org/buglist.cgi',
+    'url': 'wkb.ug',
+    'user_func': user_to_webkit_email,
+  },
+]
+
+git_instances = [
+  {
+    'option': 'webkit_repo',
+    'change_re':
+        r'git-svn-id: http://svn\.webkit\.org/repository/webkit/trunk@(\d*)',
+    'change_url': 'trac.webkit.org/changeset',
+    'review_re': r'https://bugs\.webkit\.org/show_bug\.cgi\?id\=(\d*)',
+    'review_url': 'wkb.ug',
+    'review_prop': 'webkit_bug_id',
+
+    'owner_search_func': user_to_webkit_owner_search,
+    'reviewer_search_func': user_to_webkit_reviewer_search,
+  },
+]
 
 # Uses ClientLogin to authenticate the user for Google Code issue trackers.
 def get_auth_token(email):
   # KeyringCreds will use the system keyring on the first try, and prompt for
   # a password on the next ones.
   creds = upload.KeyringCreds('code.google.com', 'code.google.com', email)
   for _ in xrange(3):
     email, password = creds.GetUserCredentials()
     url = 'https://www.google.com/accounts/ClientLogin'
     data = urllib.urlencode({
@@ skipping 66 matching lines @@
   def __init__(self, options):
     self.options = options
     self.modified_after = options.begin
     self.modified_before = options.end
     self.user = options.user
     self.changes = []
     self.reviews = []
     self.issues = []
     self.check_cookies()
     self.google_code_auth_token = None
+    self.webkit_repo = options.webkit_repo
+    if self.webkit_repo:
+      self.setup_webkit_info()
 
   # Check the codereview cookie jar to determine which Rietveld instances to
   # authenticate to.
   def check_cookies(self):
     cookie_file = os.path.expanduser('~/.codereview_upload_cookies')
     cookie_jar = cookielib.MozillaCookieJar(cookie_file)
     if not os.path.exists(cookie_file):
       exit(1)
 
     try:
@@ skipping 248 matching lines @@
     ret = []
     for entry in replies['feed']['entry']:
       e = {}
       e['created'] = datetime_from_google_code(entry['published']['$t'])
       e['content'] = entry['content']['$t']
       e['author'] = entry['author'][0]['name']['$t']
       ret.append(e)
     return ret
 
   @staticmethod
+  def git_cmd(repo, *args):
+    cmd = ['git', '--git-dir=%s/.git' % repo]
+    cmd.extend(args)
+    [stdout, _] = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE).communicate()
+    lines = str(stdout).split('\n')[:-1]
+    return lines
+
+  def git_search(self, instance, owner=None, reviewer=None):
+    repo = getattr(self, instance['option'])
+    if not repo:
+      return []
+
+    search = []
+    if owner:
+      search.extend(instance['owner_search_func'](owner))
+    if reviewer:
+      search.extend(instance['reviewer_search_func'](reviewer))
+    if not len(search):
+      return []
+
+    self.git_cmd(repo, 'fetch', 'origin')
+
+    time_format = '%Y-%m-%d %H:%M:%S'
+    log_args = [
+      '--after=' + self.modified_after.strftime(time_format),
+      '--before=' + self.modified_before.strftime(time_format),
+      '--format=%H'
+    ]
+    commits = set()
+    for query in search:
+      query_args = [query]
+      query_args.extend(log_args)
+      commits |= set(self.git_cmd(repo, 'log', 'origin/master', *query_args))
+
+    ret = []
+    for commit in commits:
+      output = self.git_cmd(repo, 'log', commit + "^!", "--format=%cn%n%cd%n%B")
+      author = output[0]
+      date = datetime.strptime(output[1], "%a %b %d %H:%M:%S %Y +0000")
+      ret.append(self.process_git_commit(instance, author, date, output[2:]))
+
+    ret = sorted(ret, key=lambda i: i['modified'], reverse=True)
+    return ret
+
+  @staticmethod
+  def process_git_commit(instance, author, date, log):
+    ret = {}
+    ret['owner'] = author
+    ret['author'] = author
+    ret['modified'] = date
+    ret['created'] = date
+    ret['header'] = log[0]
+
+    reviews = []
+    reviewers = []
+    changes = []
+
+    for line in log:
+      match = re.match(r'Reviewed by ([^.]*)', line)
+      if match:
+        reviewers.append(match.group(1))
+      if instance['review_re']:
+        match = re.match(instance['review_re'], line)
+        if match:
+          reviews.append(int(match.group(1)))
+      if instance['change_re']:
+        match = re.match(instance['change_re'], line)
+        if match:
+          changes.append(int(match.group(1)))
+
+    # TODO(enne): should convert full names to usernames via CommitterList.
+    ret['reviewers'] = set(reviewers)
+
+    # Reviews more useful than change link itself, but tricky if multiple
+    # Reviews == bugs for WebKit changes
+    if len(reviews) == 1:
+      url = 'http://%s/%d' % (instance['review_url'], reviews[0])
+      if instance['review_prop']:
+        ret[instance['review_prop']] = reviews[0]
+    else:
+      url = 'http://%s/%d' % (instance['change_url'], changes[0])
+    ret['review_url'] = url
+
+    return ret
+
+  def bugzilla_issues(self, instance, user):
+    if instance['user_func']:
+      user = instance['user_func'](user)
+    if not user:
+      return []
+
+    # This search is a little iffy, as it returns any bug that has been
+    # modified over a time period in any way and that a user has ever commented
+    # on, but that's the best that Bugzilla can get us.  Oops.
+    commented = { 'emaillongdesc1': 1 }
+    issues = self.bugzilla_search(instance, user, commented)
+    issues = filter(lambda issue: issue['owner'] != user, issues)
+
+    reported = { 'emailreporter1': 1, 'chfield': '[Bug creation]' }
+    issues.extend(self.bugzilla_search(instance, user, reported))
+
+    # Remove duplicates by bug id
+    seen = {}
+    pruned = []
+    for issue in issues:
+      bug_id = issue['webkit_bug_id']
+      if bug_id in seen:
+        continue
+      seen[bug_id] = True
+      pruned.append(issue)
+
+    # Bugzilla has no modified time, so sort by id?
+    pruned = sorted(pruned, key=lambda i: i['webkit_bug_id'])
+    return issues
+
+  def bugzilla_search(self, instance, user, params):
+    time_format = '%Y-%m-%d'
+    values = {
+      'chfieldfrom': self.modified_after.strftime(time_format),
+      'chfieldto': self.modified_before.strftime(time_format),
+      'ctype': 'csv',
+      'emailtype1': 'substring',
+      'email1': '%s' % user,
+    }
+    values.update(params)
+
+    # Must be GET not POST
+    data = urllib.urlencode(values)
+    req = urllib2.Request("%s?%s" % (instance['search_url'], data))
+    response = urllib2.urlopen(req)
+    reader = csv.reader(response)
+    reader.next() # skip the header line
+
+    issues = map(partial(self.process_bugzilla_issue, instance), reader)
+    return issues
+
+  @staticmethod
+  def process_bugzilla_issue(instance, issue):
+    bug_id, owner, desc = int(issue[0]), issue[4], issue[7]
+
+    ret = {}
+    ret['owner'] = owner
+    ret['author'] = owner
+    ret['review_url'] = 'http://%s/%d' % (instance['url'], bug_id)
+    ret['url'] = ret['review_url']
+    ret['header'] = desc
+    ret['webkit_bug_id'] = bug_id
+    return ret
+
+  def setup_webkit_info(self):
+    assert(self.webkit_repo)
+    git_dir = os.path.normpath(self.webkit_repo + "/.git")
+    if not os.path.exists(git_dir):
+      print "%s doesn't exist, turning off WebKit checks." % git_dir
+      self.webkit_repo = None
+      return
+
+    try:
+      self.git_cmd(self.webkit_repo, "fetch", "origin")
+    except subprocess.CalledProcessError:
+      print "Failed to update WebKit repo, turning off WebKit checks."
+      self.webkit_repo = None
+      return
+
+    path = "Tools/Scripts"
+    full_path = os.path.normpath("%s/%s" % (self.options.webkit_repo, path))
+    sys.path.append(full_path)
+
+    try:
+      global webkitpy
+      webkitpy = __import__('webkitpy.common.config.committers')
+    except ImportError:
+      print "Failed to import WebKit committer list, turning off WebKit checks."
+      self.webkit_repo = None
+      return
+
+    if not webkit_account(self.user):
+      email = self.user + "@chromium.org"
+      print "No %s in committers.py, turning off WebKit checks." % email
+      self.webkit_repo = None
+
+  @staticmethod
   def print_change(change):
     print '%s %s' % (
         change['review_url'],
         change['header'],
         )
 
   @staticmethod
   def print_issue(issue):
     print '%s %s' % (
         issue['url'],
@@ skipping 37 matching lines @@
     self.google_code_auth_token = (
         get_auth_token(self.options.local_user + '@chromium.org'))
 
   def get_changes(self):
     for instance in rietveld_instances:
       self.changes += self.rietveld_search(instance, owner=self.user)
 
     for instance in gerrit_instances:
       self.changes += self.gerrit_search(instance, owner=self.user)
 
+    for instance in git_instances:
+      self.changes += self.git_search(instance, owner=self.user)
+
   def print_changes(self):
     if self.changes:
       print '\nChanges:'
       for change in self.changes:
         self.print_change(change)
 
   def get_reviews(self):
     for instance in rietveld_instances:
       self.reviews += self.rietveld_search(instance, reviewer=self.user)
 
     for instance in gerrit_instances:
       reviews = self.gerrit_search(instance, reviewer=self.user)
       reviews = filter(lambda r: not username(r['owner']) == self.user, reviews)
       self.reviews += reviews
 
+    for instance in git_instances:
+      self.reviews += self.git_search(instance, reviewer=self.user)
+
   def print_reviews(self):
     if self.reviews:
       print '\nReviews:'
       for review in self.reviews:
         self.print_change(review)
 
   def get_issues(self):
     for project in google_code_projects:
       self.issues += self.google_code_issue_search(project)
 
+    for instance in bugzilla_instances:
+      self.issues += self.bugzilla_issues(instance, self.user)
+
+  def process_activities(self):
+    # If a webkit bug was a review, don't list it as an issue.
+    ids = {}
+    for review in self.reviews + self.changes:
+      if 'webkit_bug_id' in review:
+        ids[review['webkit_bug_id']] = True
+
+    def duplicate_issue(issue):
+      if 'webkit_bug_id' not in issue:
+        return False
+      return issue['webkit_bug_id'] in ids
+
+    self.issues = filter(lambda issue: not duplicate_issue(issue), self.issues)
+
   def print_issues(self):
     if self.issues:
       print '\nIssues:'
       for c in self.issues:
         self.print_issue(c)
 
   def print_activity(self):
     self.print_changes()
     self.print_reviews()
     self.print_issues()
 
 
 def main():
   # Silence upload.py.
   rietveld.upload.verbosity = 0
 
   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
   parser.add_option(
       '-u', '--user', metavar='<email>',
       default=os.environ.get('USER'),
       help='Filter on user, default=%default')
   parser.add_option(
+      '--webkit_repo', metavar='<dir>',
+      default='%s' % os.environ.get('WEBKIT_DIR'),
+      help='Local path to WebKit repository, default=%default')
+  parser.add_option(
       '-b', '--begin', metavar='<date>',
       help='Filter issues created after the date')
   parser.add_option(
       '-e', '--end', metavar='<date>',
       help='Filter issues created before the date')
   quarter_begin, quarter_end = get_quarter_of(datetime.today() -
                                               relativedelta(months=2))
   parser.add_option(
       '-Q', '--last_quarter', action='store_true',
       help='Use last quarter\'s dates, e.g. %s to %s' % (
@@ skipping 78 matching lines @@
 
   print 'Looking up activity.....'
 
   if options.changes:
     my_activity.get_changes()
   if options.reviews:
     my_activity.get_reviews()
   if options.issues:
     my_activity.get_issues()
 
+  my_activity.process_activities()
+
   print '\n\n\n'
 
   my_activity.print_changes()
   my_activity.print_reviews()
   my_activity.print_issues()
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())