Modified the bisect script to work for official Chrome builds.

bisect-builds.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Snapshot Build Bisect Tool
 
 This script bisects a snapshot archive using binary search. It starts at
 a bad revision (it will try to guess HEAD) and asks for a last known-good
 revision. It will then binary search across this revision range by downloading,
 unzipping, and opening Chromium for you. After testing the specific revision,
 it will ask you whether it is good or bad before continuing the search.
 """
 
 # The root URL for storage.
 BASE_URL = 'http://commondatastorage.googleapis.com/chromium-browser-snapshots'
 
+# The root URL for official builds.
+OFFICIAL_BASE_URL = 'http://chrome4linux.mtv.corp.google.com/archives'
+
 # URL to the ViewVC commit page.
 BUILD_VIEWVC_URL = 'http://src.chromium.org/viewvc/chrome?view=rev&revision=%d'
 
 # Changelogs URL.
 CHANGELOG_URL = 'http://build.chromium.org/f/chromium/' \
                 'perf/dashboard/ui/changelog.html?url=/trunk/src&range=%d:%d'

[Robert Sesek, 2012/05/23 21:24:01]

nit: blank line after

[anantha, 2012/05/24 18:32:11]

Done.

+# Official Changelogs URL.
+OFFICIAL_CHANGELOG_URL = 'http://omahaproxy.appspot.com/'\
+                         'changelog?old_version=%s&new_version=%s'
 
 # DEPS file URL.
 DEPS_FILE= 'http://src.chromium.org/viewvc/chrome/trunk/src/DEPS?revision=%d'
-
 # WebKit Changelogs URL.
 WEBKIT_CHANGELOG_URL = 'http://trac.webkit.org/log/' \
                        'trunk/?rev=%d&stop_rev=%d&verbose=on'
 
 ###############################################################################
 
 import math
 import optparse
 import os
 import pipes
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 import threading
 import urllib
+from distutils.version import LooseVersion
 from xml.etree import ElementTree
 import zipfile
 
 
 class PathContext(object):
   """A PathContext is used to carry the information used to construct URLs and
   paths when dealing with the storage server and archives."""
-  def __init__(self, platform, good_revision, bad_revision):
+  def __init__(self, platform, good_revision, bad_revision, is_official):
     super(PathContext, self).__init__()
     # Store off the input parameters.
     self.platform = platform  # What's passed in to the '-a/--archive' option.
     self.good_revision = good_revision
     self.bad_revision = bad_revision
+    self.is_official = is_official
 
     # The name of the ZIP file in a revision directory on the server.
     self.archive_name = None
 
     # Set some internal members:
     #   _listing_platform_dir = Directory that holds revisions. Ends with a '/'.
     #   _archive_extract_dir = Uncompressed directory in the archive_name file.
     #   _binary_name = The name of the executable to run.
     if self.platform == 'linux' or self.platform == 'linux64':
-      self._listing_platform_dir = 'Linux/'
-      self.archive_name = 'chrome-linux.zip'
-      self._archive_extract_dir = 'chrome-linux'
       self._binary_name = 'chrome'
-      # Linux and x64 share all the same path data except for the archive dir.
-      if self.platform == 'linux64':
-        self._listing_platform_dir = 'Linux_x64/'
     elif self.platform == 'mac':
-      self._listing_platform_dir = 'Mac/'
       self.archive_name = 'chrome-mac.zip'
       self._archive_extract_dir = 'chrome-mac'
-      self._binary_name = 'Chromium.app/Contents/MacOS/Chromium'
     elif self.platform == 'win':
-      self._listing_platform_dir = 'Win/'
       self.archive_name = 'chrome-win32.zip'
       self._archive_extract_dir = 'chrome-win32'
       self._binary_name = 'chrome.exe'
     else:
       raise Exception('Invalid platform: %s' % self.platform)
 
+    if is_official:
+      if self.platform == 'linux':
+         self._listing_platform_dir = 'lucid32bit/'
+         self.archive_name = 'chrome-lucid32bit.zip'
+         self._archive_extract_dir = 'chrome-lucid32bit'
+      elif self.platform == 'linux64':
+         self._listing_platform_dir = 'lucid64bit/'
+         self.archive_name = 'chrome-lucid64bit.zip'
+         self._archive_extract_dir = 'chrome-lucid64bit'
+      elif self.platform == 'mac':
+         self._listing_platform_dir = 'mac/'
+         self._binary_name = 'Google Chrome.app/Contents/MacOS/Google Chrome'
+      elif self.platform == 'win':
+         self._listing_platform_dir = 'win/'
+    else:
+      if self.platform == 'linux' or self.platform == 'linux64':
+         self.archive_name = 'chrome-linux.zip'
+         self._archive_extract_dir = 'chrome-linux'
+         if self.platform == 'linux':
+           self._listing_platform_dir = 'Linux/'
+         elif self.platform == 'linux64':
+           self._listing_platform_dir = 'Linux_x64/'
+      elif self.platform == 'mac':
+        self._listing_platform_dir = 'Mac/'
+        self._binary_name = 'Chromium.app/Contents/MacOS/Chromium'
+      elif self.platform == 'win':
+        self._listing_platform_dir = 'Win/'
+
   def GetListingURL(self, marker=None):
     """Returns the URL for a directory listing, with an optional marker."""
     marker_param = ''
     if marker:
       marker_param = '&marker=' + str(marker)
     return BASE_URL + '/?delimiter=/&prefix=' + self._listing_platform_dir + \
         marker_param
 
   def GetDownloadURL(self, revision):
     """Gets the download URL for a build archive of a specific revision."""
-    return "%s/%s%d/%s" % (
-        BASE_URL, self._listing_platform_dir, revision, self.archive_name)
+    if self.is_official:
+      return "%s/%s/%s%s" % (
+          OFFICIAL_BASE_URL, revision, self._listing_platform_dir,
+          self.archive_name)
+    else:
+      return "%s/%s%s/%s" % (
+          BASE_URL, self._listing_platform_dir, revision, self.archive_name)
 
   def GetLastChangeURL(self):
     """Returns a URL to the LAST_CHANGE file."""
     return BASE_URL + '/' + self._listing_platform_dir + 'LAST_CHANGE'
 
   def GetLaunchPath(self):
     """Returns a relative path (presumably from the archive extraction location)
     that is used to run the executable."""
     return os.path.join(self._archive_extract_dir, self._binary_name)
 
@@ skipping 34 matching lines @@
       # trailing slash to just have a number.
       revisions = []
       for prefix in all_prefixes:
         revnum = prefix.text[prefix_len:-1]
         try:
           revnum = int(revnum)
           revisions.append(revnum)
         except ValueError:
           pass
       return (revisions, next_marker)
-
     # Fetch the first list of revisions.
     (revisions, next_marker) = _FetchAndParse(self.GetListingURL())
 
     # If the result list was truncated, refetch with the next marker. Do this
     # until an entire directory listing is done.
     while next_marker:
       next_url = self.GetListingURL(next_marker)
       (new_revisions, next_marker) = _FetchAndParse(next_url)
       revisions.extend(new_revisions)
-
     return revisions
 
   def GetRevList(self):
     """Gets the list of revision numbers between self.good_revision and
     self.bad_revision."""
     # Download the revlist and filter for just the range between good and bad.
     minrev = self.good_revision
     maxrev = self.bad_revision
     revlist = map(int, self.ParseDirectoryIndex())
-    revlist = [x for x in revlist if x >= minrev and x <= maxrev]
+    revlist = [x for x in revlist if x >= int(minrev) and x <= int(maxrev)]
     revlist.sort()
     return revlist
 
+  def GetOfficialBuildsList(self):
+    """Gets the list of official build numbers between self.good_revision and
+    self.bad_revision."""
+    # Download the revlist and filter for just the range between good and bad.
+    minrev = self.good_revision
+    maxrev = self.bad_revision
+    handle = urllib.urlopen(OFFICIAL_BASE_URL)
+    dirindex = handle.read()
+    handle.close()
+    build_numbers = re.findall(r'<a href="([0-9][0-9].*)/">', dirindex)
+    final_list = []
+    start_index = '0'
+    end_index = '0'
+    i = 0
+
+    parsed_build_numbers = [LooseVersion(x) for x in build_numbers]
+    for build_number in sorted(parsed_build_numbers):
+      path = OFFICIAL_BASE_URL + '/' + str(build_number) + '/' + \
+             self._listing_platform_dir + self.archive_name
+      i = i+1

[Robert Sesek, 2012/05/23 21:24:01]

nit: spaces around operators

[anantha, 2012/05/24 18:32:11]

Done.

+      try:
+        connection = urllib.urlopen(path)

[Robert Sesek, 2012/05/23 21:24:01]

What's this for? To test that the file exists?

[anantha, 2012/05/24 18:32:11]

Correct.
On 2012/05/23 21:24:01, rsesek wrote:

+        connection.close()
+        final_list.append(str(build_number))
+        if str(build_number) == minrev:
+          start_index = i
+        if str(build_number) == maxrev:
+          end_index = i
+      except urllib.HTTPError, e:
+        print e.getcode()

[Robert Sesek, 2012/05/23 21:24:01]

This won't be a helpful error message.

[anantha, 2012/05/24 18:32:11]

changed it to pass.
On 2012/05/23 21:24:01, rsesek wrote:

+    return final_list[start_index:end_index]
 
 def UnzipFilenameToDir(filename, dir):
   """Unzip |filename| to directory |dir|."""
   cwd = os.getcwd()
   if not os.path.isabs(filename):
     filename = os.path.join(cwd, filename)
   zf = zipfile.ZipFile(filename)
   # Make base.
   try:
     if not os.path.isdir(dir):
@@ skipping 26 matching lines @@
   @param rev The Chromium revision number/tag to download.
   @param filename The destination for the downloaded file.
   @param quit_event A threading.Event which will be set by the master thread to
                     indicate that the download should be aborted.
   @param progress_event A threading.Event which will be set by the master thread
                     to indicate that the progress of the download should be
                     displayed.
   """
   def ReportHook(blocknum, blocksize, totalsize):
     if quit_event and quit_event.isSet():
-     raise RuntimeError("Aborting download of revision %d" % rev)
+      raise RuntimeError("Aborting download of revision %s" % str(rev))
     if progress_event and progress_event.isSet():
       size = blocknum * blocksize
       if totalsize == -1:  # Total size not known.
         progress = "Received %d bytes" % size
       else:
         size = min(totalsize, size)
         progress = "Received %d of %d bytes, %.2f%%" % (
             size, totalsize, 100.0 * size / totalsize)
       # Send a \r to let all progress messages use just one line of output.
       sys.stdout.write("\r" + progress)
       sys.stdout.flush()
 
   download_url = context.GetDownloadURL(rev)
   try:
     urllib.urlretrieve(download_url, filename, ReportHook)
     if progress_event and progress_event.isSet():
       print
   except RuntimeError, e:
     pass
 
 
 def RunRevision(context, revision, zipfile, profile, num_runs, args):
   """Given a zipped revision, unzip it and run the test."""
-  print "Trying revision %d..." % revision
+  print "Trying revision %s..." % str(revision)
 
   # Create a temp directory and unzip the revision into it.
   cwd = os.getcwd()
   tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
   UnzipFilenameToDir(zipfile, tempdir)
   os.chdir(tempdir)
 
-  # Run the build as many times as specified.
-  testargs = [context.GetLaunchPath(), '--user-data-dir=%s' % profile] + args
+  if context.platform == 'linux' or context.platform == 'linux64':

[Robert Sesek, 2012/05/23 21:24:01]

This can be simplified to just:

testargs = [context.GetLaunchPath(), '--user-data-dir=%s' % profile] + args
# The sandbox must be run as root on Official Chrome, so bypass it.
if context.is_official and (context.platform == 'linux' or context.platform ==
'linux64'):
  testargs.append('--no-sandbox')

[anantha, 2012/05/24 18:32:11]

Done.

+    if context.is_official:
+      # Run the build as many times as specified.
+      testargs = [context.GetLaunchPath(), '--no-sandbox',
+                  '--user-data-dir=%s' % profile] + args
+    else:
+      # Run the build as many times as specified.
+      testargs = [context.GetLaunchPath(), '--user-data-dir=%s' % profile] + \
+                  args
+  else:
+    # Run the build as many times as specified.
+    testargs = [context.GetLaunchPath(), '--user-data-dir=%s' % profile] + args
+
   for i in range(0, num_runs):
     subproc = subprocess.Popen(testargs,
                                bufsize=-1,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
     (stdout, stderr) = subproc.communicate()
 
   os.chdir(cwd)
   try:
     shutil.rmtree(tempdir, True)
   except Exception, e:
     pass
 
   return (subproc.returncode, stdout, stderr)
 
 
-def AskIsGoodBuild(rev, status, stdout, stderr):
+def AskIsGoodBuild(rev, official_builds, status, stdout, stderr):
   """Ask the user whether build |rev| is good or bad."""
   # Loop until we get a response that we can parse.
   while True:
-    response = raw_input('Revision %d is [(g)ood/(b)ad/(q)uit]: ' % int(rev))
+    response = raw_input('Revision %s is [(g)ood/(b)ad/(q)uit]: ' % str(rev))
     if response and response in ('g', 'b'):
       return response == 'g'
     if response and response == 'q':
       raise SystemExit()
 
 
 def Bisect(platform,
+           official_builds,
            good_rev=0,
            bad_rev=0,
            num_runs=1,
            try_args=(),
            profile=None,
            predicate=AskIsGoodBuild):
   """Given known good and known bad revisions, run a binary search on all
   archived revisions to determine the last known good revision.
 
   @param platform Which build to download/run ('mac', 'win', 'linux64', etc.).
   @param good_rev Number/tag of the last known good revision.

[Robert Sesek, 2012/05/23 21:24:01]

Need a @param offiical_builds ... before this

[anantha, 2012/05/24 18:32:11]

Done.

   @param bad_rev Number/tag of the first known bad revision.
   @param num_runs Number of times to run each build for asking good/bad.
   @param try_args A tuple of arguments to pass to the test application.
   @param profile The name of the user profile to run with.
   @param predicate A predicate function which returns True iff the argument
                    chromium revision is good.
 
   Threading is used to fetch Chromium revisions in the background, speeding up
   the user's experience. For example, suppose the bounds of the search are
   good_rev=0, bad_rev=100. The first revision to be checked is 50. Depending on
   whether revision 50 is good or bad, the next revision to check will be either
   25 or 75. So, while revision 50 is being checked, the script will download
   revisions 25 and 75 in the background. Once the good/bad verdict on rev 50 is
   known:
 
     - If rev 50 is good, the download of rev 25 is cancelled, and the next test
       is run on rev 75.
 
     - If rev 50 is bad, the download of rev 75 is cancelled, and the next test
       is run on rev 25.
   """
 
   if not profile:
     profile = 'profile'
 
-  context = PathContext(platform, good_rev, bad_rev)
+  context = PathContext(platform, good_rev, bad_rev, official_builds)
   cwd = os.getcwd()
 
-  _GetDownloadPath = lambda rev: os.path.join(cwd,
-      '%d-%s' % (rev, context.archive_name))
+
 
   print "Downloading list of known revisions..."
-
-  revlist = context.GetRevList()
+  _GetDownloadPath = lambda rev: os.path.join(cwd,
+    '%s-%s' % (str(rev), context.archive_name))

[Robert Sesek, 2012/05/23 21:24:01]

nit: continuations are indented 4

[anantha, 2012/05/24 18:32:11]

Done.

+  if official_builds:
+    revlist = context.GetOfficialBuildsList()
+  else:
+    revlist = context.GetRevList()
 
   # Get a list of revisions to bisect across.
   if len(revlist) < 2:  # Don't have enough builds to bisect.
     msg = 'We don\'t have enough builds to bisect. revlist: %s' % revlist
     raise RuntimeError(msg)
 
   # Figure out our bookends and first pivot point; fetch the pivot revision.
   good = 0
   bad = len(revlist) - 1
   pivot = bad / 2
   rev = revlist[pivot]
   zipfile = _GetDownloadPath(rev)
   progress_event = threading.Event()
   progress_event.set()
-  print "Downloading revision %d..." % rev
+  print "Downloading revision %s..." % str(rev)
   FetchRevision(context, rev, zipfile,
                 quit_event=None, progress_event=progress_event)
 
   # Binary search time!
   while zipfile and bad - good > 1:
     # Pre-fetch next two possible pivots
     #   - down_pivot is the next revision to check if the current revision turns
     #     out to be bad.
     #   - up_pivot is the next revision to check if the current revision turns
     #     out to be good.
@@ skipping 38 matching lines @@
                                            profile,
                                            num_runs,
                                            try_args)
     os.unlink(zipfile)
     zipfile = None
 
     # Call the predicate function to see if the current revision is good or bad.
     # On that basis, kill one of the background downloads and complete the
     # other, as described in the comments above.
     try:
-      if predicate(rev, status, stdout, stderr):
+      if predicate(rev, official_builds, status, stdout, stderr):
         good = pivot
         if down_thread:
           down_quit_event.set()  # Kill the download of older revision.
           down_thread.join()
           os.unlink(down_zipfile)
         if up_thread:
-          print "Downloading revision %d..." % up_rev
+          print "Downloading revision %s..." % str(up_rev)
           up_progress_event.set()  # Display progress of download.
           up_thread.join()  # Wait for newer revision to finish downloading.
           pivot = up_pivot
           zipfile = up_zipfile
       else:
         bad = pivot
         if up_thread:
           up_quit_event.set()  # Kill download of newer revision.
           up_thread.join()
           os.unlink(up_zipfile)
         if down_thread:
-          print "Downloading revision %d..." % down_rev
+          print "Downloading revision %s..." % str(down_rev)
           down_progress_event.set()  # Display progress of download.
           down_thread.join()  # Wait for older revision to finish downloading.
           pivot = down_pivot
           zipfile = down_zipfile
     except SystemExit:
       print "Cleaning up..."
       for f in [_GetDownloadPath(revlist[down_pivot]),
                 _GetDownloadPath(revlist[up_pivot])]:
         try:
           os.unlink(f)
@@ skipping 26 matching lines @@
            '\n'
            'Tip: add "-- --no-first-run" to bypass the first run prompts.')
   parser = optparse.OptionParser(usage=usage)
   # Strangely, the default help output doesn't include the choice list.
   choices = ['mac', 'win', 'linux', 'linux64']
             # linux-chromiumos lacks a continuous archive http://crbug.com/78158
   parser.add_option('-a', '--archive',
                     choices = choices,
                     help = 'The buildbot archive to bisect [%s].' %
                            '|'.join(choices))
-  parser.add_option('-b', '--bad', type = 'int',
+  parser.add_option('-o', action="store_true", dest='official_builds',
+                    help = 'Specify if it is a chromium or official build.')

[Robert Sesek, 2012/05/23 21:24:01]

help = 'Bisect across official Chrome builds (internal only) instead of Chromium
archives.'

[anantha, 2012/05/24 18:32:11]

Done.

+  parser.add_option('-b', '--bad', type = 'str',
                     help = 'The bad revision to bisect to.')
-  parser.add_option('-g', '--good', type = 'int',
+  parser.add_option('-g', '--good', type = 'str',
                     help = 'The last known good revision to bisect from.')
   parser.add_option('-p', '--profile', '--user-data-dir', type = 'str',
                     help = 'Profile to use; this will not reset every run. ' +
                     'Defaults to a clean profile.', default = 'profile')
   parser.add_option('-t', '--times', type = 'int',
                     help = 'Number of times to run each build before asking ' +
                     'if it\'s good or bad. Temporary profiles are reused.',
                     default = 1)
   (opts, args) = parser.parse_args()
 
   if opts.archive is None:
     print 'Error: missing required parameter: --archive'
     print
     parser.print_help()
     return 1
 
   if opts.bad and opts.good and (opts.good > opts.bad):
     print ('The good revision (%d) must precede the bad revision (%d).\n' %
            (opts.good, opts.bad))
     parser.print_help()
     return 1
 
   # Create the context. Initialize 0 for the revisions as they are set below.
-  context = PathContext(opts.archive, 0, 0)
+  context = PathContext(opts.archive, 0, 0, opts.official_builds)
+
+  if opts.official_builds and opts.bad is None:
+    print ' Needs a bad build range'
+    print
+    parser.print_help()
+    return 1
 
   # Pick a starting point, try to get HEAD for this.
   if opts.bad:
     bad_rev = opts.bad
   else:
     bad_rev = 0
     try:
       # Location of the latest build revision number
       nh = urllib.urlopen(context.GetLastChangeURL())
       latest = int(nh.read())
@@ skipping 16 matching lines @@
     except Exception, e:
       pass
 
   if opts.times < 1:
     print('Number of times to run (%d) must be greater than or equal to 1.' %
           opts.times)
     parser.print_help()
     return 1
 
   (last_known_good_rev, first_known_bad_rev) = Bisect(
-      opts.archive, good_rev, bad_rev, opts.times, args, opts.profile)
+      opts.archive, opts.official_builds, good_rev, bad_rev, opts.times, args,
+      opts.profile)
 
   # Get corresponding webkit revisions.
   try:
     last_known_good_webkit_rev = GetWebKitRevisionForChromiumRevision(
         last_known_good_rev)
     first_known_bad_webkit_rev = GetWebKitRevisionForChromiumRevision(
         first_known_bad_rev)
   except Exception, e:
     # Silently ignore the failure.
     last_known_good_webkit_rev, first_known_bad_webkit_rev = 0, 0
 
   # We're done. Let the user know the results in an official manner.
-  print('You are probably looking for build %d.' % first_known_bad_rev)
+  print('You are probably looking for build %s.' % str(first_known_bad_rev))
+
   if last_known_good_webkit_rev != first_known_bad_webkit_rev:
     print 'WEBKIT CHANGELOG URL:'
     print WEBKIT_CHANGELOG_URL % (first_known_bad_webkit_rev,
                                   last_known_good_webkit_rev)
   print 'CHANGELOG URL:'
-  print CHANGELOG_URL % (last_known_good_rev, first_known_bad_rev)
-  print 'Built at revision:'
-  print BUILD_VIEWVC_URL % first_known_bad_rev
+  if opts.official_builds:
+    print OFFICIAL_CHANGELOG_URL % (last_known_good_rev, first_known_bad_rev)
+  else:
+    print CHANGELOG_URL % (last_known_good_rev, first_known_bad_rev)
+    print 'Built at revision:'
+    print BUILD_VIEWVC_URL % first_known_bad_rev
 
 
 if __name__ == '__main__':
-  sys.exit(main())
+  sys.exit(main())