Adding gather_stats (based on bisect-builds.py) to gather benchmarking

tools/gpu/gather_stats.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Snapshot Build Linear Benchmark Tool
+
+This script executes and gathers stats from a series of snapshot archives. It
+starts at a start revision (it will try to guess HEAD) and asks for an end
+revision. It will then linearly run across this revision range by downloading,
+unzipping, and opening Chromium for you. After testing the specific revision,
+it will continuing the process.

[Ian Vollick, 2012/09/18 17:46:32]

Does this script do all the guessing and asking that is claimed here? I'd ditch
the last sentence, and in the sentence previous I'd s/opening Chromium for
you/gathers stats for each revision/

[hartmanng, 2012/09/19 02:17:55]

Done.

+"""
+
+import optparse
+import os
+import sys
+import tempfile
+from distutils.version import LooseVersion
+
+sys.path.insert(0, "..")

[Ian Vollick, 2012/09/18 17:46:32]

Does this make an assumption about where the script is run from?

[hartmanng, 2012/09/19 02:17:55]

Done.

+bisect_builds = __import__('bisect-builds')
+
+def gather_stats(platform,
+                 official_builds,
+                 execute,
+                 start_rev=0,
+                 end_rev=0,
+                 num_runs=1,
+                 stdoutdir=None):
+  """Given start and known end revisions, run sequentially
+  on archived revisions, gathering performance stats at each one.
+
+  @param platform Which build to download/run ('mac', 'win', 'linux64', etc.).
+  @param official_builds Specify build type (Chromium or Official build).
+  @param execute String specifying what command to run at each revision.
+  @param start_rev Number/tag of the revision to start on.
+  @param end_rev Number/tag of the revision to end on.
+  @param num_runs Number of times to run each build.
+  @param stdoutdir The (optional) path to an output directory."""
+
+  context = bisect_builds.PathContext(platform, start_rev, end_rev,
+                                      official_builds)
+  cwd = os.getcwd()
+
+
+  print "Downloading list of known revisions..."
+  _GetDownloadPath = lambda rev: os.path.join(cwd,
+      '%s-%s' % (str(rev), context.archive_name))
+  if official_builds:
+    revlist = context.GetOfficialBuildsList()
+  else:
+    revlist = context.GetRevList()
+
+  print 'Running on range [%s, %s].' % (revlist[0], revlist[-1])
+
+  # Figure out our bookends and first pivot point; fetch the pivot revision.

[Ian Vollick, 2012/09/18 17:46:32]

Should avoid bisect-specific language now. Pivot is out, maybe pivot -> index?

[hartmanng, 2012/09/19 02:17:55]

Done.

+  rev = revlist[0]
+  next_rev = 0
+  zipfile = _GetDownloadPath(rev)
+  initial_fetch = bisect_builds.DownloadJob(context, 'initial_fetch', rev,
+                                            zipfile)
+  initial_fetch.Start()
+  initial_fetch.WaitFor()
+  pivot = 1
+
+  while zipfile and pivot < len(revlist):

[Ian Vollick, 2012/09/18 17:46:32]

Why not 'for rev in revlist:'?

Also, isn't pivot currently the index of the revision we're prefetching? It
looks like we won't enter this list and test the last revision since the
prefetch index will be too high.

[hartmanng, 2012/09/19 02:17:55]

Yes you're right, it didn't test the last revision (Fixed). However, changing
the loop to a "for in" style loop makes it difficult to perform prefetching
since we need variables pointing at 2 separate points in revlist at the same
time.

We could do "for next_rev in revlist[1:}:" since we already have rev pointing to
revlist[0] (and keep updating rev = next_rev at the end of each iteration), but
then the loop will again end too early to execute the last prefetched revision
unless we add an extra call to RunRevision after the loop.

+    # Pre-fetch next pivot
+    cur_fetch = None
+    next_rev = revlist[pivot]
+    cur_fetch = bisect_builds.DownloadJob(context, 'cur_fetch', next_rev,
+                                          _GetDownloadPath(next_rev))
+    cur_fetch.Start()
+
+    # Run test on the pivot revision.
+    tempdir = tempfile.mkdtemp(prefix='gather_stats_tmp')
+    args = (execute % os.path.join(tempdir,
+                                    context.GetLaunchPath())).split(' ')
+    print args
+    (status, stdout, stderr) = bisect_builds.RunRevision(context,
+                                                         rev,
+                                                         zipfile,
+                                                         num_runs,
+                                                         args,
+                                                         tempdir,
+                                                         stdoutdir)
+    try:
+      shutil.rmtree(tempdir, True)
+    except Exception, e:
+      pass
+
+    rev = next_rev
+
+    os.unlink(zipfile)
+    zipfile = None
+
+    # Wait for fetch to complete.
+    try:
+      if cur_fetch:
+        cur_fetch.WaitFor()
+        pivot = pivot + 1
+        zipfile = cur_fetch.zipfile
+    except SystemExit:
+      print "Cleaning up..."
+      for f in [_GetDownloadPath(revlist[pivot])]:
+        try:
+          os.unlink(f)
+        except OSError:
+          pass
+      sys.exit(0)
+
+  return 0
+
+def main():
+  usage = ('%prog [options]\n'
+           'Perform binary search on the snapshot builds.\n'

[Ian Vollick, 2012/09/18 17:46:32]

This text needs to be updated.

[hartmanng, 2012/09/19 02:17:55]

Done.

+           '\n'
+           'Tip: add "-- --no-first-run" to bypass the first run prompts.')
+  parser = optparse.OptionParser(usage=usage)
+  # Strangely, the default help output doesn't include the choice list.
+  choices = ['mac', 'win', 'linux', 'linux64']
+            # linux-chromiumos lacks a continuous archive http://crbug.com/78158
+  parser.add_option('-a', '--archive',
+                    choices = choices,
+                    help = 'The buildbot archive to run [%s].' %
+                           '|'.join(choices))
+  parser.add_option('-o', action="store_true", dest='official_builds',
+                    help = 'Run across official ' +
+                    'Chrome builds (internal only) instead of ' +
+                    'Chromium archives.')
+  parser.add_option('-e', '--end', type = 'str',
+                    help = 'The revision to end on. Default is HEAD.')
+  parser.add_option('-s', '--start', type = 'str',
+                    help = 'The revision to start from. Default is 0.')

[Ian Vollick, 2012/09/18 17:46:32]

0's a rough starting revision! It would be nice if we could also specify a time
delta, like a month, and the default was going back one month.

[hartmanng, 2012/09/19 02:17:55]

I agree, but do you know a good way to get dates of the revisions?  The current
bisect_builds.py script's revlist source (eg
http://commondatastorage.googleapis.com/chromium-browser-snapshots/?delimiter...
) doesn't seem to have easily-accessible datestamps. For now I've put a revision
delta instead of a time delta.

+  parser.add_option('-t', '--times', type = 'int',
+                    help = 'Number of times to run each build. Temporary' +
+                    ' profiles are reused. Default is 1.',
+                    default = 1)
+  parser.add_option('-x', '--execute', type='str',
+                    help = 'Command to execute for each revision. Include a ' +
+                    'single %s to specify the location of the downloaded ' +
+                    'chrome executable. Default is \'python ' +
+                    'run_scrolling_benchmark page_sets/Q32012.json ' +
+                    '--browser-executable=%s\'.',
+                    default = 'python ./run_scrolling_benchmark ' +
+                    'page_sets/Q32012.json --browser-executable=%s')
+  parser.add_option('-d', '--output-directory', '--stdoutdir', type='str',
+                    help = 'Save stdout to files for each revision.',
+                    default = None)
+  (opts, args) = parser.parse_args()
+
+  if opts.archive is None:
+    print 'Error: missing required parameter: --archive'
+    print
+    parser.print_help()
+    return 1
+
+  # Create the context. Initialize 0 for the revisions as they are set below.
+  context = bisect_builds.PathContext(opts.archive, 0, 0, opts.official_builds)
+  # Pick a starting point, try to get HEAD for this.
+  if opts.end:
+    end_rev = opts.end
+  else:
+    end_rev = '999.0.0.0'
+    if not opts.official_builds:
+      end_rev = GetChromiumRevision(context.GetLastChangeURL())
+
+  # Find out when we were good.
+  if opts.start:
+    start_rev = opts.start
+  else:
+    start_rev = '0.0.0.0' if opts.official_builds else 0
+
+  if opts.official_builds:
+    start_rev = LooseVersion(start_rev)
+    end_rev = LooseVersion(end_rev)
+  else:
+    start_rev = int(start_rev)
+    end_rev = int(end_rev)
+
+  if start_rev > end_rev:
+    print ('The start revision (%s) must precede the end revision (%s).\n' %
+           (start_rev, end_rev))
+    parser.print_help()
+    return 1
+
+  if opts.times < 1:
+    print('Number of times to run (%d) must be greater than or equal to 1.' %
+          opts.times)
+    parser.print_help()
+    return 1
+
+  return gather_stats(opts.archive, opts.official_builds, opts.execute,
+                      start_rev, end_rev, opts.times, opts.output_directory)
+
+if __name__ == '__main__':
+  sys.exit(main())