Added gsutil to depot tools

third_party/gsutil/gslib/commands/perfdiag.py

+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains the perfdiag gsutil command."""
+
+import calendar
+from collections import defaultdict
+import contextlib
+import datetime
+import json
+import math
+import multiprocessing
+import os
+import re
+import socket
+import string
+import subprocess
+import tempfile
+import time
+
+import boto.gs.connection
+
+from gslib.command import Command
+from gslib.command import COMMAND_NAME
+from gslib.command import COMMAND_NAME_ALIASES
+from gslib.command import CONFIG_REQUIRED
+from gslib.command import FILE_URIS_OK
+from gslib.command import MAX_ARGS
+from gslib.command import MIN_ARGS
+from gslib.command import PROVIDER_URIS_OK
+from gslib.command import SUPPORTED_SUB_ARGS
+from gslib.command import URIS_START_ARG
+from gslib.commands import config
+from gslib.exception import CommandException
+from gslib.help_provider import HELP_NAME
+from gslib.help_provider import HELP_NAME_ALIASES
+from gslib.help_provider import HELP_ONE_LINE_SUMMARY
+from gslib.help_provider import HELP_TEXT
+from gslib.help_provider import HELP_TYPE
+from gslib.help_provider import HelpType
+from gslib.util import IS_LINUX
+from gslib.util import MakeBitsHumanReadable
+from gslib.util import MakeHumanReadable
+from gslib.util import Percentile
+
+_detailed_help_text = ("""
+<B>SYNOPSIS</B>
+  gsutil perfdiag [-i in.json] [-o out.json]
+                  [-n iterations] [-c concurrency] [-s size] [-t tests] uri...
+
+
+<B>DESCRIPTION</B>
+  The perfdiag command runs a suite of diagnostic tests for a given Google
+  Storage bucket.
+
+  The 'uri' parameter must name an existing bucket (e.g. gs://foo) to which
+  the user has write permission. Several test files will be uploaded to and
+  downloaded from this bucket. All test files will be deleted at the completion
+  of the diagnostic if it finishes successfully.
+
+  gsutil performance can be impacted by many factors at the client, server,
+  and in-between, such as: CPU speed; available memory; the access path to the
+  local disk; network bandwidth; contention and error rates along the path
+  between gsutil and Google; operating system buffering configuration; and
+  firewalls and other network elements. The perfdiag command is provided so
+  that customers can run a known measurement suite when troubleshooting
+  performance problems.
+
+
+<B>PROVIDING DIAGNOSTIC OUTPUT TO GOOGLE CLOUD STORAGE TEAM</B>
+  If the Google Cloud Storage Team asks you to run a performance diagnostic
+  please use the following command, and email the output file (output.json)
+  to gs-team@google.com:
+
+    gsutil perfdiag -o output.json gs://your-bucket
+
+
+<B>OPTIONS</B>
+  -n          Sets the number of iterations performed when downloading and
+              uploading files during latency and throughput tests. Defaults to
+              5.
+
+  -c          Sets the level of concurrency to use while running throughput
+              experiments. The default value of 1 will only run a single read
+              or write operation concurrently.
+
+  -s          Sets the size (in bytes) of the test file used to perform read
+              and write throughput tests. The default is 1 MiB.
+
+  -t          Sets the list of diagnostic tests to perform. The default is to
+              run all diagnostic tests. Must be a comma-separated list
+              containing one or more of the following:
+
+                  lat: Runs N iterations (set with -n) of writing the file,
+                       retrieving its metadata, reading the file, and deleting
+                       the file. Records the latency of each operation.
+
+                rthru: Runs N (set with -n) read operations, with at most C
+                       (set with -c) reads outstanding at any given time.
+
+                wthru: Runs N (set with -n) write operations, with at most C
+                       (set with -c) writes outstanding at any given time.
+
+  -o          Writes the results of the diagnostic to an output file. The output
+              is a JSON file containing system information and performance
+              diagnostic results. The file can be read and reported later using
+              the -i option.
+
+  -i          Reads the JSON output file created using the -o command and prints
+              a formatted description of the results.
+
+
+<B>NOTE</B>
+  The perfdiag command collects system information. It collects your IP address,
+  executes DNS queries to Google servers and collects the results, and collects
+  network statistics information from the output of netstat -s. None of this
+  information will be sent to Google unless you choose to send it.
+""")
+
+
+class PerfDiagCommand(Command):
+  """Implementation of gsutil perfdiag command."""
+
+  # Command specification (processed by parent class).
+  command_spec = {
+      # Name of command.
+      COMMAND_NAME: 'perfdiag',
+      # List of command name aliases.
+      COMMAND_NAME_ALIASES: ['diag', 'diagnostic', 'perf', 'performance'],
+      # Min number of args required by this command.
+      MIN_ARGS: 0,
+      # Max number of args required by this command, or NO_MAX.
+      MAX_ARGS: 1,
+      # Getopt-style string specifying acceptable sub args.
+      SUPPORTED_SUB_ARGS: 'n:c:s:t:i:o:',
+      # True if file URIs acceptable for this command.
+      FILE_URIS_OK: False,
+      # True if provider-only URIs acceptable for this command.
+      PROVIDER_URIS_OK: False,
+      # Index in args of first URI arg.
+      URIS_START_ARG: 0,
+      # True if must configure gsutil before running command.
+      CONFIG_REQUIRED: True,
+  }
+  help_spec = {
+      # Name of command or auxiliary help info for which this help applies.
+      HELP_NAME: 'perfdiag',
+      # List of help name aliases.
+      HELP_NAME_ALIASES: [],
+      # Type of help:
+      HELP_TYPE: HelpType.COMMAND_HELP,
+      # One line summary of this help.
+      HELP_ONE_LINE_SUMMARY: 'Run performance diagnostic',
+      # The full help text.
+      HELP_TEXT: _detailed_help_text,
+  }
+
+  # Byte sizes to use for testing files.
+  # TODO: Consider letting the user specify these sizes with a configuration
+  # parameter.
+  test_file_sizes = (
+      0,  # 0 bytes
+      1024,  # 1 KB
+      102400,  # 100 KB
+      1048576,  # 1MB
+  )
+
+  # List of all diagnostic tests.
+  ALL_DIAG_TESTS = ('rthru', 'wthru', 'lat')
+
+  # Google Cloud Storage API endpoint host.
+  GOOGLE_API_HOST = boto.gs.connection.GSConnection.DefaultHost
+
+  def _WindowedExec(self, cmd, n, w, raise_on_error=True):
+    """Executes a command n times with a window size of w.
+
+    Up to w instances of the command will be executed and left outstanding at a
+    time until n instances of the command have completed.
+
+    Args:
+      cmd: List containing the command to execute.
+      n: Number of times the command will be executed.
+      w: Window size of outstanding commands being executed.
+      raise_on_error: See _Exec.
+
+    Raises:
+      Exception: If raise_on_error is set to True and any process exits with a
+      non-zero return code.
+    """
+    if self.debug:
+      print 'Running command:', cmd
+    devnull_f = open(os.devnull, 'w')
+    num_finished = 0
+    running = []
+    while len(running) or num_finished < n:
+      # Fires off new commands that can be executed.
+      while len(running) < w and num_finished + len(running) < n:
+        print 'Starting concurrent command: %s' % (' '.join(cmd))
+        p = subprocess.Popen(cmd, stdout=devnull_f, stderr=devnull_f)
+        running.append(p)
+
+      # Checks for finished commands.
+      prev_running = running
+      running = []
+      for p in prev_running:
+        retcode = p.poll()
+        if retcode is None:
+          running.append(p)
+        elif raise_on_error and retcode:
+          raise CommandException("Received non-zero return code (%d) from "
+                                 "subprocess '%s'." % (retcode, ' '.join(cmd)))
+        else:
+          num_finished += 1
+
+  def _Exec(self, cmd, raise_on_error=True, return_output=False,
+            mute_stderr=False):
+    """Executes a command in a subprocess.
+
+    Args:
+      cmd: List containing the command to execute.
+      raise_on_error: Whether or not to raise an exception when a process exits
+          with a non-zero return code.
+      return_output: If set to True, the return value of the function is the
+          stdout of the process.
+      mute_stderr: If set to True, the stderr of the process is not printed to
+          the console.
+
+    Returns:
+      The return code of the process or the stdout if return_output is set.
+
+    Raises:
+      Exception: If raise_on_error is set to True and any process exits with a
+      non-zero return code.
+    """
+    if self.debug:
+      print 'Running command:', cmd
+    stderr = subprocess.PIPE if mute_stderr else None
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr)
+    (stdoutdata, stderrdata) = p.communicate()
+    if raise_on_error and p.returncode:
+      raise CommandException("Received non-zero return code (%d) from "
+                             "subprocess '%s'." % (p.returncode, ' '.join(cmd)))
+    return stdoutdata if return_output else p.returncode
+
+  def _GsUtil(self, cmd, raise_on_error=True, return_output=False,
+              mute_stderr=False):
+    """Executes a gsutil command in a subprocess.
+
+    Args:
+      cmd: A list containing the arguments to the gsutil program, e.g. ['ls',
+          'gs://foo'].
+      raise_on_error: see _Exec.
+      return_output: see _Exec.
+      mute_stderr: see _Exec.
+
+    Returns:
+      The return code of the process or the stdout if return_output is set.
+    """
+    cmd = self.gsutil_exec_list + cmd
+    return self._Exec(cmd, raise_on_error=raise_on_error,
+                      return_output=return_output, mute_stderr=mute_stderr)
+
+  def _SetUp(self):
+    """Performs setup operations needed before diagnostics can be run."""
+
+    # Stores test result data.
+    self.results = {}
+    # List of test files in a temporary location on disk for latency ops.
+    self.latency_files = []
+    # Maps each test file path to its size in bytes.
+    self.file_sizes = {}
+    # Maps each test file to its contents as a string.
+    self.file_contents = {}
+
+    def _MakeFile(file_size):
+      """Creates a temporary file of the given size and returns its path."""
+      fd, fpath = tempfile.mkstemp(suffix='.bin', prefix='gsutil_test_file',
+                                   text=False)
+      self.file_sizes[fpath] = file_size
+      f = os.fdopen(fd, 'wb')
+      f.write(os.urandom(file_size))
+      f.close()
+      f = open(fpath, 'rb')
+      self.file_contents[fpath] = f.read()
+      f.close()
+      return fpath
+
+    # Create files for latency tests.
+    for file_size in self.test_file_sizes:
+      fpath = _MakeFile(file_size)
+      self.latency_files.append(fpath)
+
+    # Local file on disk for write throughput tests.
+    self.thru_local_file = _MakeFile(self.thru_filesize)
+    # Remote file to write/read from during throughput tests.
+    self.thru_remote_file = (str(self.bucket_uri) +
+                             os.path.basename(self.thru_local_file))
+
+  def _TearDown(self):
+    """Performs operations to clean things up after performing diagnostics."""
+    for fpath in self.latency_files + [self.thru_local_file]:
+      try:
+        os.remove(fpath)
+      except OSError:
+        pass
+
+    self._GsUtil(['rm', self.thru_remote_file], raise_on_error=False,
+                 mute_stderr=True)
+
+  @contextlib.contextmanager
+  def _Time(self, key, bucket):
+    """A context manager that measures time.
+
+    A context manager that prints a status message before and after executing
+    the inner command and times how long the inner command takes. Keeps track of
+    the timing, aggregated by the given key.
+
+    Args:
+      key: The key to insert the timing value into a dictionary bucket.
+      bucket: A dictionary to place the timing value in.
+
+    Yields:
+      For the context manager.
+    """
+    print key, 'starting...'
+    t0 = time.time()
+    yield
+    t1 = time.time()
+    bucket[key].append(t1 - t0)
+    print key, 'done.'
+
+  def _RunLatencyTests(self):
+    """Runs latency tests."""
+    # Stores timing information for each category of operation.
+    self.results['latency'] = defaultdict(list)
+
+    for i in range(self.num_iterations):
+      print
+      print 'Running latency iteration %d...' % (i+1)
+      for fpath in self.latency_files:
+        basename = os.path.basename(fpath)
+        gsbucket = str(self.bucket_uri)
+        gsuri = gsbucket + basename
+        file_size = self.file_sizes[fpath]
+        readable_file_size = MakeHumanReadable(file_size)
+
+        print
+        print ("File of size %(size)s located on disk at '%(fpath)s' being "
+               "diagnosed in the cloud at '%(gsuri)s'."
+               % {'size': readable_file_size,
+                  'fpath': fpath,
+                  'gsuri': gsuri})
+
+        k = self.bucket.key_class(self.bucket)
+        k.key = basename
+
+        with self._Time('UPLOAD_%d' % file_size, self.results['latency']):
+          k.set_contents_from_string(self.file_contents[fpath])
+        with self._Time('METADATA_%d' % file_size, self.results['latency']):
+          k.exists()
+        with self._Time('DOWNLOAD_%d' % file_size, self.results['latency']):
+          k.get_contents_as_string()
+        with self._Time('DELETE_%d' % file_size, self.results['latency']):
+          k.delete()
+
+  def _RunReadThruTests(self):
+    """Runs read throughput tests."""
+    self.results['read_throughput'] = {'file_size': self.thru_filesize,
+                                       'num_times': self.num_iterations,
+                                       'concurrency': self.concurrency}
+
+    # Copy the file to remote location before reading.
+    self._GsUtil(['cp', self.thru_local_file, self.thru_remote_file])
+
+    if self.concurrency == 1:
+      k = self.bucket.key_class(self.bucket)
+      k.key = os.path.basename(self.thru_local_file)
+      # Warm up the TCP connection by transferring a couple times first.
+      for i in range(2):
+        k.get_contents_as_string()
+      t0 = time.time()
+      for i in range(self.num_iterations):
+        k.get_contents_as_string()
+      t1 = time.time()
+    else:
+      cmd = self.gsutil_exec_list + ['cp', self.thru_remote_file, os.devnull]
+      t0 = time.time()
+      self._WindowedExec(cmd, self.num_iterations, self.concurrency)
+      t1 = time.time()
+
+    time_took = t1 - t0
+    total_bytes_copied = self.thru_filesize * self.num_iterations
+    bytes_per_second = total_bytes_copied / time_took
+
+    self.results['read_throughput']['time_took'] = time_took
+    self.results['read_throughput']['total_bytes_copied'] = total_bytes_copied
+    self.results['read_throughput']['bytes_per_second'] = bytes_per_second
+
+  def _RunWriteThruTests(self):
+    """Runs write throughput tests."""
+    self.results['write_throughput'] = {'file_size': self.thru_filesize,
+                                        'num_copies': self.num_iterations,
+                                        'concurrency': self.concurrency}
+
+    if self.concurrency == 1:
+      k = self.bucket.key_class(self.bucket)
+      k.key = os.path.basename(self.thru_local_file)
+      # Warm up the TCP connection by transferring a couple times first.
+      for i in range(2):
+        k.set_contents_from_string(self.file_contents[self.thru_local_file])
+      t0 = time.time()
+      for i in range(self.num_iterations):
+        k.set_contents_from_string(self.file_contents[self.thru_local_file])
+      t1 = time.time()
+    else:
+      cmd = self.gsutil_exec_list + ['cp', self.thru_local_file,
+                                     self.thru_remote_file]
+      t0 = time.time()
+      self._WindowedExec(cmd, self.num_iterations, self.concurrency)
+      t1 = time.time()
+
+    time_took = t1 - t0
+    total_bytes_copied = self.thru_filesize * self.num_iterations
+    bytes_per_second = total_bytes_copied / time_took
+
+    self.results['write_throughput']['time_took'] = time_took
+    self.results['write_throughput']['total_bytes_copied'] = total_bytes_copied
+    self.results['write_throughput']['bytes_per_second'] = bytes_per_second
+
+  def _GetDiskCounters(self):
+    """Retrieves disk I/O statistics for all disks.
+
+    Adapted from the psutil module's psutil._pslinux.disk_io_counters:
+      http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py
+
+    Originally distributed under under a BSD license.
+    Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola.
+
+    Returns:
+      A dictionary containing disk names mapped to the disk counters from
+      /disk/diskstats.
+    """
+    # iostat documentation states that sectors are equivalent with blocks and
+    # have a size of 512 bytes since 2.4 kernels. This value is needed to
+    # calculate the amount of disk I/O in bytes.
+    sector_size = 512
+
+    partitions = []
+    with open('/proc/partitions', 'r') as f:
+      lines = f.readlines()[2:]
+      for line in lines:
+        _, _, _, name = line.split()
+        if name[-1].isdigit():
+          partitions.append(name)
+
+    retdict = {}
+    with open('/proc/diskstats', 'r') as f:
+      for line in f:
+        values = line.split()[:11]
+        _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values
+        if name in partitions:
+          rbytes = int(rbytes) * sector_size
+          wbytes = int(wbytes) * sector_size
+          reads = int(reads)
+          writes = int(writes)
+          rtime = int(rtime)
+          wtime = int(wtime)
+          retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime)
+    return retdict
+
+  def _GetTcpStats(self):
+    """Tries to parse out TCP packet information from netstat output.
+
+    Returns:
+       A dictionary containing TCP information
+    """
+    # netstat return code is non-zero for -s on Linux, so don't raise on error.
+    netstat_output = self._Exec(['netstat', '-s'], return_output=True,
+                                raise_on_error=False)
+    netstat_output = netstat_output.strip().lower()
+    found_tcp = False
+    tcp_retransmit = None
+    tcp_received = None
+    tcp_sent = None
+    for line in netstat_output.split('\n'):
+      # Header for TCP section is "Tcp:" in Linux/Mac and
+      # "TCP Statistics for" in Windows.
+      if 'tcp:' in line or 'tcp statistics' in line:
+        found_tcp = True
+
+      # Linux == "segments retransmited" (sic), Mac == "retransmit timeouts"
+      # Windows == "segments retransmitted".
+      if (found_tcp and tcp_retransmit is None and
+          ('segments retransmited' in line or 'retransmit timeouts' in line or
+           'segments retransmitted' in line)):
+        tcp_retransmit = ''.join(c for c in line if c in string.digits)
+
+      # Linux+Windows == "segments received", Mac == "packets received".
+      if (found_tcp and tcp_received is None and
+          ('segments received' in line or 'packets received' in line)):
+        tcp_received = ''.join(c for c in line if c in string.digits)
+
+      # Linux == "segments send out" (sic), Mac+Windows == "packets sent".
+      if (found_tcp and tcp_sent is None and
+          ('segments send out' in line or 'packets sent' in line or
+           'segments sent' in line)):
+        tcp_sent = ''.join(c for c in line if c in string.digits)
+
+    result = {}
+    try:
+      result['tcp_retransmit'] = int(tcp_retransmit)
+      result['tcp_received'] = int(tcp_received)
+      result['tcp_sent'] = int(tcp_sent)
+    except (ValueError, TypeError):
+      result['tcp_retransmit'] = None
+      result['tcp_received'] = None
+      result['tcp_sent'] = None
+
+    return result
+
+  def _CollectSysInfo(self):
+    """Collects system information."""
+    sysinfo = {}
+
+    # Get the local IP address from socket lib.
+    sysinfo['ip_address'] = socket.gethostbyname(socket.gethostname())
+    # Record the temporary directory used since it can affect performance, e.g.
+    # when on a networked filesystem.
+    sysinfo['tempdir'] = tempfile.gettempdir()
+
+    # Produces an RFC 2822 compliant GMT timestamp.
+    sysinfo['gmt_timestamp'] = time.strftime('%a, %d %b %Y %H:%M:%S +0000',
+                                             time.gmtime())
+
+    # Execute a CNAME lookup on Google DNS to find what Google server
+    # it's routing to.
+    cmd = ['nslookup', '-type=CNAME', self.GOOGLE_API_HOST]
+    nslookup_cname_output = self._Exec(cmd, return_output=True)
+    m = re.search(r' = (?P<googserv>[^.]+)\.', nslookup_cname_output)
+    sysinfo['googserv_route'] = m.group('googserv') if m else None
+
+    # Look up IP addresses for Google Server.
+    (hostname, aliaslist, ipaddrlist) = socket.gethostbyname_ex(
+        self.GOOGLE_API_HOST)
+    sysinfo['googserv_ips'] = ipaddrlist
+
+    # Reverse lookup the hostnames for the Google Server IPs.
+    sysinfo['googserv_hostnames'] = []
+    for googserv_ip in ipaddrlist:
+      (hostname, aliaslist, ipaddrlist) = socket.gethostbyaddr(googserv_ip)
+      sysinfo['googserv_hostnames'].append(hostname)
+
+    # Query o-o to find out what the Google DNS thinks is the user's IP.
+    cmd = ['nslookup', '-type=TXT', 'o-o.myaddr.google.com.']
+    nslookup_txt_output = self._Exec(cmd, return_output=True)
+    m = re.search(r'text\s+=\s+"(?P<dnsip>[\.\d]+)"', nslookup_txt_output)
+    sysinfo['dns_o-o_ip'] = m.group('dnsip') if m else None
+
+    # Try and find the number of CPUs in the system if available.
+    try:
+      sysinfo['cpu_count'] = multiprocessing.cpu_count()
+    except NotImplementedError:
+      sysinfo['cpu_count'] = None
+
+    # For *nix platforms, obtain the CPU load.
+    try:
+      sysinfo['load_avg'] = list(os.getloadavg())
+    except (AttributeError, OSError):
+      sysinfo['load_avg'] = None
+
+    # Try and collect memory information from /proc/meminfo if possible.
+    mem_total = None
+    mem_free = None
+    mem_buffers = None
+    mem_cached = None
+
+    try:
+      with open('/proc/meminfo', 'r') as f:
+        for line in f:
+          if line.startswith('MemTotal'):
+            mem_total = (int(''.join(c for c in line if c in string.digits))
+                         * 1000)
+          elif line.startswith('MemFree'):
+            mem_free = (int(''.join(c for c in line if c in string.digits))
+                        * 1000)
+          elif line.startswith('Buffers'):
+            mem_buffers = (int(''.join(c for c in line if c in string.digits))
+                           * 1000)
+          elif line.startswith('Cached'):
+            mem_cached = (int(''.join(c for c in line if c in string.digits))
+                          * 1000)
+    except (IOError, ValueError):
+      pass
+
+    sysinfo['meminfo'] = {'mem_total': mem_total,
+                          'mem_free': mem_free,
+                          'mem_buffers': mem_buffers,
+                          'mem_cached': mem_cached}
+
+    # Get configuration attributes from config module.
+    sysinfo['gsutil_config'] = {}
+    for attr in dir(config):
+      attr_value = getattr(config, attr)
+      # Filter out multiline strings that are not useful.
+      if attr.isupper() and not (isinstance(attr_value, basestring) and
+                                 '\n' in attr_value):
+        sysinfo['gsutil_config'][attr] = attr_value
+
+    self.results['sysinfo'] = sysinfo
+
+  def _DisplayStats(self, trials):
+    """Prints out mean, standard deviation, median, and 90th percentile."""
+    n = len(trials)
+    mean = float(sum(trials)) / n
+    stdev = math.sqrt(sum((x - mean)**2 for x in trials) / n)
+
+    print str(n).rjust(6), '',
+    print ('%.1f' % (mean * 1000)).rjust(9), '',
+    print ('%.1f' % (stdev * 1000)).rjust(12), '',
+    print ('%.1f' % (Percentile(trials, 0.5) * 1000)).rjust(11), '',
+    print ('%.1f' % (Percentile(trials, 0.9) * 1000)).rjust(11), ''
+
+  def _DisplayResults(self):
+    """Displays results collected from diagnostic run."""
+    print
+    print '=' * 78
+    print 'DIAGNOSTIC RESULTS'.center(78)
+    print '=' * 78
+
+    if 'latency' in self.results:
+      print
+      print '-' * 78
+      print 'Latency'.center(78)
+      print '-' * 78
+      print ('Operation       Size  Trials  Mean (ms)  Std Dev (ms)  '
+             'Median (ms)  90th % (ms)')
+      print ('=========  =========  ======  =========  ============  '
+             '===========  ===========')
+      for key in sorted(self.results['latency']):
+        trials = sorted(self.results['latency'][key])
+        op, numbytes = key.split('_')
+        numbytes = int(numbytes)
+        if op == 'METADATA':
+          print 'Metadata'.rjust(9), '',
+          print MakeHumanReadable(numbytes).rjust(9), '',
+          self._DisplayStats(trials)
+        if op == 'DOWNLOAD':
+          print 'Download'.rjust(9), '',
+          print MakeHumanReadable(numbytes).rjust(9), '',
+          self._DisplayStats(trials)
+        if op == 'UPLOAD':
+          print 'Upload'.rjust(9), '',
+          print MakeHumanReadable(numbytes).rjust(9), '',
+          self._DisplayStats(trials)
+        if op == 'DELETE':
+          print 'Delete'.rjust(9), '',
+          print MakeHumanReadable(numbytes).rjust(9), '',
+          self._DisplayStats(trials)
+
+    if 'write_throughput' in self.results:
+      print
+      print '-' * 78
+      print 'Write Throughput'.center(78)
+      print '-' * 78
+      write_thru = self.results['write_throughput']
+      print 'Copied a %s file %d times for a total transfer size of %s.' % (
+          MakeHumanReadable(write_thru['file_size']),
+          write_thru['num_copies'],
+          MakeHumanReadable(write_thru['total_bytes_copied']))
+      print 'Write throughput: %s/s.' % (
+          MakeBitsHumanReadable(write_thru['bytes_per_second'] * 8))
+
+    if 'read_throughput' in self.results:
+      print
+      print '-' * 78
+      print 'Read Throughput'.center(78)
+      print '-' * 78
+      read_thru = self.results['read_throughput']
+      print 'Copied a %s file %d times for a total transfer size of %s.' % (
+          MakeHumanReadable(read_thru['file_size']),
+          read_thru['num_times'],
+          MakeHumanReadable(read_thru['total_bytes_copied']))
+      print 'Read throughput: %s/s.' % (
+          MakeBitsHumanReadable(read_thru['bytes_per_second'] * 8))
+
+    if 'sysinfo' in self.results:
+      print
+      print '-' * 78
+      print 'System Information'.center(78)
+      print '-' * 78
+      info = self.results['sysinfo']
+      print 'IP Address: \n  %s' % info['ip_address']
+      print 'Temporary Directory: \n  %s' % info['tempdir']
+      print 'Bucket URI: \n  %s' % self.results['bucket_uri']
+
+      if 'gmt_timestamp' in info:
+        ts_string = info['gmt_timestamp']
+        timetuple = None
+        try:
+          # Convert RFC 2822 string to Linux timestamp.
+          timetuple = time.strptime(ts_string, '%a, %d %b %Y %H:%M:%S +0000')
+        except ValueError:
+          pass
+
+        if timetuple:
+          # Converts the GMT time tuple to local Linux timestamp.
+          localtime = calendar.timegm(timetuple)
+          localdt = datetime.datetime.fromtimestamp(localtime)
+          print 'Measurement time: \n %s' % localdt.strftime(
+              '%Y-%m-%d %I-%M-%S %p %Z')
+
+      print 'Google Server: \n  %s' % info['googserv_route']
+      print ('Google Server IP Addresses: \n  %s' %
+             ('\n  '.join(info['googserv_ips'])))
+      print ('Google Server Hostnames: \n  %s' %
+             ('\n  '.join(info['googserv_hostnames'])))
+      print 'Google DNS thinks your IP is: \n  %s' % info['dns_o-o_ip']
+      print 'CPU Count: \n  %s' % info['cpu_count']
+      print 'CPU Load Average: \n  %s' % info['load_avg']
+      try:
+        print ('Total Memory: \n  %s' %
+               MakeHumanReadable(info['meminfo']['mem_total']))
+        # Free memory is really MemFree + Buffers + Cached.
+        print 'Free Memory: \n  %s' % MakeHumanReadable(
+            info['meminfo']['mem_free'] +
+            info['meminfo']['mem_buffers'] +
+            info['meminfo']['mem_cached'])
+      except TypeError:
+        pass
+
+      netstat_after = info['netstat_end']
+      netstat_before = info['netstat_start']
+      for tcp_type in ('sent', 'received', 'retransmit'):
+        try:
+          delta = (netstat_after['tcp_%s' % tcp_type] -
+                   netstat_before['tcp_%s' % tcp_type])
+          print 'TCP segments %s during test:\n  %d' % (tcp_type, delta)
+        except TypeError:
+          pass
+
+      if 'disk_counters_end' in info and 'disk_counters_start' in info:
+        print 'Disk Counter Deltas:\n',
+        disk_after = info['disk_counters_end']
+        disk_before = info['disk_counters_start']
+        print '', 'disk'.rjust(6),
+        for colname in ['reads', 'writes', 'rbytes', 'wbytes', 'rtime',
+                        'wtime']:
+          print colname.rjust(8),
+        print
+        for diskname in sorted(disk_after):
+          before = disk_before[diskname]
+          after = disk_after[diskname]
+          (reads1, writes1, rbytes1, wbytes1, rtime1, wtime1) = before
+          (reads2, writes2, rbytes2, wbytes2, rtime2, wtime2) = after
+          print '', diskname.rjust(6),
+          deltas = [reads2-reads1, writes2-writes1, rbytes2-rbytes1,
+                    wbytes2-wbytes1, rtime2-rtime1, wtime2-wtime1]
+          for delta in deltas:
+            print str(delta).rjust(8),
+          print
+
+    if self.output_file:
+      with open(self.output_file, 'w') as f:
+        json.dump(self.results, f, indent=2)
+      print
+      print "Output file written to '%s'." % self.output_file
+
+    print
+
+  def _ParsePositiveInteger(self, val, msg):
+    """Tries to convert val argument to a positive integer.
+
+    Args:
+      val: The value (as a string) to convert to a positive integer.
+      msg: The error message to place in the CommandException on an error.
+
+    Returns:
+      A valid positive integer.
+
+    Raises:
+      CommandException: If the supplied value is not a valid positive integer.
+    """
+    try:
+      val = int(val)
+      if val < 1:
+        raise CommandException(msg)
+      return val
+    except ValueError:
+      raise CommandException(msg)
+
+  def _ParseArgs(self):
+    """Parses arguments for perfdiag command."""
+    # From -n.
+    self.num_iterations = 5
+    # From -c.
+    self.concurrency = 1
+    # From -s.
+    self.thru_filesize = 1048576
+    # From -t.
+    self.diag_tests = self.ALL_DIAG_TESTS
+    # From -o.
+    self.output_file = None
+    # From -i.
+    self.input_file = None
+
+    if self.sub_opts:
+      for o, a in self.sub_opts:
+        if o == '-n':
+          self.num_iterations = self._ParsePositiveInteger(
+              a, 'The -n parameter must be a positive integer.')
+        if o == '-c':
+          self.concurrency = self._ParsePositiveInteger(
+              a, 'The -c parameter must be a positive integer.')
+        if o == '-s':
+          self.thru_filesize = self._ParsePositiveInteger(
+              a, 'The -s parameter must be a positive integer.')
+        if o == '-t':
+          self.diag_tests = []
+          for test_name in a.strip().split(','):
+            if test_name.lower() not in self.ALL_DIAG_TESTS:
+              raise CommandException("List of test names (-t) contains invalid "
+                                     "test name '%s'." % test_name)
+            self.diag_tests.append(test_name)
+        if o == '-o':
+          self.output_file = os.path.abspath(a)
+        if o == '-i':
+          self.input_file = os.path.abspath(a)
+          if not os.path.isfile(self.input_file):
+            raise CommandException("Invalid input file (-i): '%s'." % a)
+          try:
+            with open(self.input_file, 'r') as f:
+              self.results = json.load(f)
+              print "Read input file: '%s'." % self.input_file
+          except ValueError:
+            raise CommandException("Could not decode input file (-i): '%s'." %
+                                   a)
+          return
+
+    if not self.args:
+      raise CommandException('Wrong number of arguments for "perfdiag" '
+                             'command.')
+    self.bucket_uri = self.suri_builder.StorageUri(self.args[0])
+    if not self.bucket_uri.names_bucket():
+      raise CommandException('The perfdiag command requires a URI that '
+                             'specifies a bucket.\n"%s" is not '
+                             'valid.' % self.bucket_uri)
+    self.bucket = self.bucket_uri.get_bucket()
+
+  # Command entry point.
+  def RunCommand(self):
+    """Called by gsutil when the command is being invoked."""
+    self._ParseArgs()
+
+    if self.input_file:
+      self._DisplayResults()
+      return 0
+
+    print 'Number of iterations to run: %d' % self.num_iterations
+    print 'Base bucket URI: %s' % self.bucket_uri
+    print 'Concurrency level: %d' % self.concurrency
+    print 'Throughput file size: %s' % MakeHumanReadable(self.thru_filesize)
+    print 'Diagnostics to run: %s' % (', '.join(self.diag_tests))
+
+    try:
+      self._SetUp()
+
+      # Collect generic system info.
+      self._CollectSysInfo()
+      # Collect netstat info and disk counters before tests (and again later).
+      self.results['sysinfo']['netstat_start'] = self._GetTcpStats()
+      if IS_LINUX:
+        self.results['sysinfo']['disk_counters_start'] = self._GetDiskCounters()
+      # Record bucket URI.
+      self.results['bucket_uri'] = str(self.bucket_uri)
+
+      if 'lat' in self.diag_tests:
+        self._RunLatencyTests()
+      if 'rthru' in self.diag_tests:
+        self._RunReadThruTests()
+      if 'wthru' in self.diag_tests:
+        self._RunWriteThruTests()
+
+      # Collect netstat info and disk counters after tests.
+      self.results['sysinfo']['netstat_end'] = self._GetTcpStats()
+      if IS_LINUX:
+        self.results['sysinfo']['disk_counters_end'] = self._GetDiskCounters()
+
+      self._DisplayResults()
+    finally:
+      self._TearDown()
+
+    return 0