Added gsutil to depot tools

third_party/gsutil/gslib/commands/cp.py

+# Copyright 2011 Google Inc. All Rights Reserved.
+# Copyright 2011, Nexenta Systems Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import boto
+import errno
+import gzip
+import hashlib
+import mimetypes
+import os
+import platform
+import re
+import subprocess
+import stat
+import sys
+import tempfile
+import threading
+import time
+
+from boto import config
+from boto.exception import GSResponseError
+from boto.exception import ResumableUploadException
+from boto.gs.resumable_upload_handler import ResumableUploadHandler
+from boto.s3.keyfile import KeyFile
+from boto.s3.resumable_download_handler import ResumableDownloadHandler
+from boto.storage_uri import BucketStorageUri
+from gslib.command import COMMAND_NAME
+from gslib.command import COMMAND_NAME_ALIASES
+from gslib.command import CONFIG_REQUIRED
+from gslib.command import Command
+from gslib.command import FILE_URIS_OK
+from gslib.command import MAX_ARGS
+from gslib.command import MIN_ARGS
+from gslib.command import PROVIDER_URIS_OK
+from gslib.command import SUPPORTED_SUB_ARGS
+from gslib.command import URIS_START_ARG
+from gslib.exception import CommandException
+from gslib.help_provider import HELP_NAME
+from gslib.help_provider import HELP_NAME_ALIASES
+from gslib.help_provider import HELP_ONE_LINE_SUMMARY
+from gslib.help_provider import HELP_TEXT
+from gslib.help_provider import HELP_TYPE
+from gslib.help_provider import HelpType
+from gslib.name_expansion import NameExpansionIterator
+from gslib.util import ExtractErrorDetail
+from gslib.util import IS_WINDOWS
+from gslib.util import MakeHumanReadable
+from gslib.util import NO_MAX
+from gslib.util import TWO_MB
+from gslib.wildcard_iterator import ContainsWildcard
+
+_detailed_help_text = ("""
+<B>SYNOPSIS</B>
+  gsutil cp [OPTION]... src_uri dst_uri
+    - or -
+  gsutil cp [OPTION]... src_uri... dst_uri
+    - or -
+  gsutil cp [OPTION]... -I dst_uri
+
+
+<B>DESCRIPTION</B>
+  The gsutil cp command allows you to copy data between your local file
+  system and the cloud, copy data within the cloud, and copy data between
+  cloud storage providers. For example, to copy all text files from the
+  local directory to a bucket you could do:
+
+    gsutil cp *.txt gs://my_bucket
+
+  Similarly, you can download text files from a bucket by doing:
+
+    gsutil cp gs://my_bucket/*.txt .
+
+  If you want to copy an entire directory tree you need to use the -R option:
+
+    gsutil cp -R dir gs://my_bucket
+
+  If you have a large number of files to upload you might want to use the
+  gsutil -m option, to perform a parallel (multi-threaded/multi-processing)
+  copy:
+
+    gsutil -m cp -R dir gs://my_bucket
+
+  You can pass a list of URIs to copy on STDIN instead of as command line
+  arguments by using the -I option. This allows you to use gsutil in a
+  pipeline to copy files and objects as generated by a program, such as:
+
+    some_program | gsutil -m cp -I gs://my_bucket
+
+  The contents of STDIN can name files, cloud URIs, and wildcards of files
+  and cloud URIs.
+
+
+<B>HOW NAMES ARE CONSTRUCTED</B>
+  The gsutil cp command strives to name objects in a way consistent with how
+  Linux cp works, which causes names to be constructed in varying ways depending
+  on whether you're performing a recursive directory copy or copying
+  individually named objects; and whether you're copying to an existing or
+  non-existent directory.
+
+  When performing recursive directory copies, object names are constructed
+  that mirror the source directory structure starting at the point of
+  recursive processing. For example, the command:
+
+    gsutil cp -R dir1/dir2 gs://my_bucket
+
+  will create objects named like gs://my_bucket/dir2/a/b/c, assuming
+  dir1/dir2 contains the file a/b/c.
+
+  In contrast, copying individually named files will result in objects named
+  by the final path component of the source files. For example, the command:
+
+    gsutil cp dir1/dir2/** gs://my_bucket
+
+  will create objects named like gs://my_bucket/c.
+
+  The same rules apply for downloads: recursive copies of buckets and
+  bucket subdirectories produce a mirrored filename structure, while copying
+  individually (or wildcard) named objects produce flatly named files.
+
+  Note that in the above example the '**' wildcard matches all names
+  anywhere under dir. The wildcard '*' will match names just one level deep. For
+  more details see 'gsutil help wildcards'.
+
+  There's an additional wrinkle when working with subdirectories: the resulting
+  names depend on whether the destination subdirectory exists. For example,
+  if gs://my_bucket/subdir exists as a subdirectory, the command:
+
+    gsutil cp -R dir1/dir2 gs://my_bucket/subdir
+
+  will create objects named like gs://my_bucket/subdir/dir2/a/b/c. In contrast,
+  if gs://my_bucket/subdir does not exist, this same gsutil cp command will
+  create objects named like gs://my_bucket/subdir/a/b/c.
+
+
+<B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B>
+  You can use gsutil to copy to and from subdirectories by using a command like:
+
+    gsutil cp -R dir gs://my_bucket/data
+
+  This will cause dir and all of its files and nested subdirectories to be
+  copied under the specified destination, resulting in objects with names like
+  gs://my_bucket/data/dir/a/b/c. Similarly you can download from bucket
+  subdirectories by using a command like:
+
+    gsutil cp -R gs://my_bucket/data dir
+
+  This will cause everything nested under gs://my_bucket/data to be downloaded
+  into dir, resulting in files with names like dir/data/a/b/c.
+
+  Copying subdirectories is useful if you want to add data to an existing
+  bucket directory structure over time. It's also useful if you want
+  to parallelize uploads and downloads across multiple machines (often
+  reducing overall transfer time compared with simply running gsutil -m
+  cp on one machine). For example, if your bucket contains this structure:
+
+    gs://my_bucket/data/result_set_01/
+    gs://my_bucket/data/result_set_02/
+    ...
+    gs://my_bucket/data/result_set_99/
+
+  you could perform concurrent downloads across 3 machines by running these
+  commands on each machine, respectively:
+
+    gsutil -m cp -R gs://my_bucket/data/result_set_[0-3]* dir
+    gsutil -m cp -R gs://my_bucket/data/result_set_[4-6]* dir
+    gsutil -m cp -R gs://my_bucket/data/result_set_[7-9]* dir
+
+  Note that dir could be a local directory on each machine, or it could
+  be a directory mounted off of a shared file server; whether the latter
+  performs acceptably may depend on a number of things, so we recommend
+  you experiment and find out what works best for you.
+
+
+<B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B>
+  If both the source and destination URI are cloud URIs from the same
+  provider, gsutil copies data "in the cloud" (i.e., without downloading
+  to and uploading from the machine where you run gsutil). In addition to
+  the performance and cost advantages of doing this, copying in the cloud
+  preserves metadata (like Content-Type and Cache-Control).  In contrast,
+  when you download data from the cloud it ends up in a file, which has
+  no associated metadata. Thus, unless you have some way to hold on to
+  or re-create that metadata, downloading to a file will not retain the
+  metadata.
+
+  Note that by default, the gsutil cp command does not copy the object
+  ACL to the new object, and instead will use the default bucket ACL (see
+  "gsutil help setdefacl").  You can override this behavior with the -p
+  option (see OPTIONS below).
+
+  gsutil does not preserve metadata when copying objects between providers.
+
+
+<B>RESUMABLE TRANSFERS</B>
+  gsutil automatically uses the Google Cloud Storage resumable upload
+  feature whenever you use the cp command to upload an object that is larger
+  than 2 MB. You do not need to specify any special command line options
+  to make this happen. If your upload is interrupted you can restart the
+  upload by running the same cp command that you ran to start the upload.
+
+  Similarly, gsutil automatically performs resumable downloads (using HTTP
+  standard Range GET operations) whenever you use the cp command to download an
+  object larger than 2 MB.
+
+  Resumable uploads and downloads store some state information in a file
+  in ~/.gsutil named by the destination object or file. If you attempt to
+  resume a transfer from a machine with a different directory, the transfer
+  will start over from scratch.
+
+  See also "gsutil help prod" for details on using resumable transfers
+  in production.
+
+
+<B>STREAMING TRANSFERS</B>
+  Use '-' in place of src_uri or dst_uri to perform a streaming
+  transfer. For example:
+    long_running_computation | gsutil cp - gs://my_bucket/obj
+
+  Streaming transfers do not support resumable uploads/downloads.
+  (The Google resumable transfer protocol has a way to support streaming
+  transers, but gsutil doesn't currently implement support for this.)
+
+
+<B>CHANGING TEMP DIRECTORIES</B>
+  gsutil writes data to a temporary directory in several cases:
+    - when compressing data to be uploaded (see the -z option)
+    - when decompressing data being downloaded (when the data has
+      Content-Encoding:gzip, e.g., as happens when uploaded using gsutil cp -z)
+    - when running integration tests (using the gsutil test command)
+
+  In these cases it's possible the temp file location on your system that
+  gsutil selects by default may not have enough space. If you find that
+  gsutil runs out of space during one of these operations (e.g., raising
+  "CommandException: Inadequate temp space available to compress <your file>"
+  during a gsutil cp -z operation), you can change where it writes these
+  temp files by setting the TMPDIR environment variable. On Linux and MacOS
+  you can do this either by running gsutil this way:
+
+    TMPDIR=/some/directory gsutil cp ...
+
+  or by adding this line to your ~/.bashrc file and then restarting the shell
+  before running gsutil:
+
+    export TMPDIR=/some/directory
+
+  On Windows 7 you can change the TMPDIR environment variable from Start ->
+  Computer -> System -> Advanced System Settings -> Environment Variables.
+  You need to reboot after making this change for it to take effect. (Rebooting
+  is not necessary after running the export command on Linux and MacOS.)
+
+
+<B>OPTIONS</B>
+  -a canned_acl Sets named canned_acl when uploaded objects created. See
+                'gsutil help acls' for further details.
+
+  -c            If an error occurrs, continue to attempt to copy the remaining
+                files.
+
+  -D            Copy in "daisy chain" mode, i.e., copying between two buckets by
+                hooking a download to an upload, via the machine where gsutil is
+                run. By default, data are copied between two buckets "in the
+                cloud", i.e., without needing to copy via the machine where
+                gsutil runs. However, copy-in-the-cloud is not supported when
+                copying between different locations (like US and EU) or between
+                different storage classes (like STANDARD and
+                DURABLE_REDUCED_AVAILABILITY). For these cases, you can use the
+                -D option to copy data between buckets.
+                Note: Daisy chain mode is automatically used when copying
+                between providers (e.g., to copy data from Google Cloud Storage
+                to another provider).
+
+  -e            Exclude symlinks. When specified, symbolic links will not be
+                copied.
+
+  -n            No-clobber. When specified, existing files or objects at the
+                destination will not be overwritten. Any items that are skipped
+                by this option will be reported as being skipped. This option
+                will perform an additional HEAD request to check if an item
+                exists before attempting to upload the data. This will save
+                retransmitting data, but the additional HTTP requests may make
+                small object transfers slower and more expensive.
+
+                This option can be combined with the -c option to build a script
+                that copies a large number of objects, allowing retries when
+                some failures occur from which gsutil doesn't automatically
+                recover, using a bash script like the following:
+
+                    status=1
+                    while [ $status -ne 0 ] ; do
+                        gsutil cp -c -n -R ./dir gs://bucket
+                        status=$?
+                    done
+
+                The -c option will cause copying to continue after failures
+                occur, and the -n option will cause objects already copied to be
+                skipped on subsequent iterations. The loop will continue running
+                as long as gsutil exits with a non-zero status (such a status
+                indicates there was at least one failure during the gsutil run).
+
+  -p            Causes ACLs to be preserved when copying in the cloud. Note that
+                this option has performance and cost implications, because it
+                is essentially performing three requests (getacl, cp, setacl).
+                (The performance issue can be mitigated to some degree by
+                using gsutil -m cp to cause parallel copying.)
+
+                You can avoid the additional performance and cost of using cp -p
+                if you want all objects in the destination bucket to end up with
+                the same ACL by setting a default ACL on that bucket instead of
+                using cp -p. See "help gsutil setdefacl".
+
+                Note that it's not valid to specify both the -a and -p options
+                together.
+
+  -q            Causes copies to be performed quietly, i.e., without reporting
+                progress indicators of files being copied. Errors are still
+                reported. This option can be useful for running gsutil from a
+                cron job that logs its output to a file, for which the only
+                information desired in the log is failures.
+
+  -R, -r        Causes directories, buckets, and bucket subdirectories to be
+                copied recursively. If you neglect to use this option for
+                an upload, gsutil will copy any files it finds and skip any
+                directories. Similarly, neglecting to specify -R for a download
+                will cause gsutil to copy any objects at the current bucket
+                directory level, and skip any subdirectories.
+
+  -v            Requests that the version-specific URI for each uploaded object
+                be printed. Given this URI you can make future upload requests
+                that are safe in the face of concurrent updates, because Google
+                Cloud Storage will refuse to perform the update if the current
+                object version doesn't match the version-specific URI. See
+                'gsutil help versioning' for more details. Note: at present this
+                option does not work correctly for objects copied "in the cloud"
+                (e.g., gsutil cp gs://bucket/obj1 gs://bucket/obj2).
+
+  -z ext1,...   Compresses file uploads with the given extensions. If you are
+                uploading a large file with compressible content, such as
+                a .js, .css, or .html file, you can gzip-compress the file
+                during the upload process by specifying the -z <extensions>
+                option. Compressing data before upload saves on usage charges
+                because you are uploading a smaller amount of data.
+
+                When you specify the -z option, the data from your files is
+                compressed before it is uploaded, but your actual files are left
+                uncompressed on the local disk. The uploaded objects retain the
+                original content type and name as the original files but are
+                given a Content-Encoding header with the value "gzip" to
+                indicate that the object data stored are compressed on the
+                Google Cloud Storage servers.
+
+                For example, the following command:
+
+                  gsutil cp -z html -a public-read cattypes.html gs://mycats
+
+                will do all of the following:
+                  - Upload as the object gs://mycats/cattypes.html (cp command)
+                  - Set the Content-Type to text/html (based on file extension)
+                  - Compress the data in the file cattypes.html (-z option)
+                  - Set the Content-Encoding to gzip (-z option)
+                  - Set the ACL to public-read (-a option)
+                  - If a user tries to view cattypes.html in a browser, the
+                    browser will know to uncompress the data based on the
+                    Content-Encoding header, and to render it as HTML based on
+                    the Content-Type header.
+""")
+
+class CpCommand(Command):
+  """
+  Implementation of gsutil cp command.
+
+  Note that CpCommand is run for both gsutil cp and gsutil mv. The latter
+  happens by MvCommand calling CpCommand and passing the hidden (undocumented)
+  -M option. This allows the copy and remove needed for each mv to run
+  together (rather than first running all the cp's and then all the rm's, as
+  we originally had implemented), which in turn avoids the following problem
+  with removing the wrong objects: starting with a bucket containing only
+  the object gs://bucket/obj, say the user does:
+    gsutil mv gs://bucket/* gs://bucket/d.txt
+  If we ran all the cp's and then all the rm's and we didn't expand the wildcard
+  first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt,
+  and the rm command would then remove that object. In the implementation
+  prior to gsutil release 3.12 we avoided this by building a list of objects
+  to process and then running the copies and then the removes; but building
+  the list up front limits scalability (compared with the current approach
+  of processing the bucket listing iterator on the fly).
+  """
+
+  # Set default Content-Type type.
+  DEFAULT_CONTENT_TYPE = 'application/octet-stream'
+  USE_MAGICFILE = boto.config.getbool('GSUtil', 'use_magicfile', False)
+
+  # Command specification (processed by parent class).
+  command_spec = {
+    # Name of command.
+    COMMAND_NAME : 'cp',
+    # List of command name aliases.
+    COMMAND_NAME_ALIASES : ['copy'],
+    # Min number of args required by this command.
+    MIN_ARGS : 1,
+    # Max number of args required by this command, or NO_MAX.
+    MAX_ARGS : NO_MAX,
+    # Getopt-style string specifying acceptable sub args.
+    # -t is deprecated but leave intact for now to avoid breakage.
+    SUPPORTED_SUB_ARGS : 'a:cDeIMNnpqrRtvz:',
+    # True if file URIs acceptable for this command.
+    FILE_URIS_OK : True,
+    # True if provider-only URIs acceptable for this command.
+    PROVIDER_URIS_OK : False,
+    # Index in args of first URI arg.
+    URIS_START_ARG : 0,
+    # True if must configure gsutil before running command.
+    CONFIG_REQUIRED : True,
+  }
+  help_spec = {
+    # Name of command or auxiliary help info for which this help applies.
+    HELP_NAME : 'cp',
+    # List of help name aliases.
+    HELP_NAME_ALIASES : ['copy'],
+    # Type of help:
+    HELP_TYPE : HelpType.COMMAND_HELP,
+    # One line summary of this help.
+    HELP_ONE_LINE_SUMMARY : 'Copy files and objects',
+    # The full help text.
+    HELP_TEXT : _detailed_help_text,
+  }
+
+  def _CheckFinalMd5(self, key, file_name):
+    """
+    Checks that etag from server agrees with md5 computed after the
+    download completes.
+    """
+    obj_md5 = key.etag.strip('"\'')
+    file_md5 = None
+
+    if hasattr(key, 'md5') and key.md5:
+      file_md5 = key.md5
+    else:
+      print 'Computing MD5 from scratch for resumed download'
+
+      # Open file in binary mode to avoid surprises in Windows.
+      fp = open(file_name, 'rb')
+      try:
+        file_md5 = key.compute_md5(fp)[0]
+      finally:
+        fp.close()
+
+    if self.debug:
+      print 'Checking file md5 against etag. (%s/%s)' % (file_md5, obj_md5)
+    if file_md5 != obj_md5:
+      # Checksums don't match - remove file and raise exception.
+      os.unlink(file_name)
+      raise CommandException(
+        'File changed during download: md5 signature doesn\'t match '
+        'etag (incorrect downloaded file deleted)')
+
+  def _CheckForDirFileConflict(self, exp_src_uri, dst_uri):
+    """Checks whether copying exp_src_uri into dst_uri is not possible.
+
+       This happens if a directory exists in local file system where a file
+       needs to go or vice versa. In that case we print an error message and
+       exits. Example: if the file "./x" exists and you try to do:
+         gsutil cp gs://mybucket/x/y .
+       the request can't succeed because it requires a directory where
+       the file x exists.
+
+       Note that we don't enforce any corresponding restrictions for buckets,
+       because the flat namespace semantics for buckets doesn't prohibit such
+       cases the way hierarchical file systems do. For example, if a bucket
+       contains an object called gs://bucket/dir and then you run the command:
+         gsutil cp file1 file2 gs://bucket/dir
+       you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and
+       gs://bucket/dir/file2.
+
+    Args:
+      exp_src_uri: Expanded source StorageUri of copy.
+      dst_uri: Destination URI.
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    if dst_uri.is_cloud_uri():
+      # The problem can only happen for file destination URIs.
+      return
+    dst_path = dst_uri.object_name
+    final_dir = os.path.dirname(dst_path)
+    if os.path.isfile(final_dir):
+      raise CommandException('Cannot retrieve %s because a file exists '
+                             'where a directory needs to be created (%s).' %
+                             (exp_src_uri, final_dir))
+    if os.path.isdir(dst_path):
+      raise CommandException('Cannot retrieve %s because a directory exists '
+                             '(%s) where the file needs to be created.' %
+                             (exp_src_uri, dst_path))
+
+  def _InsistDstUriNamesContainer(self, exp_dst_uri,
+                                  have_existing_dst_container, command_name):
+    """
+    Raises an exception if URI doesn't name a directory, bucket, or bucket
+    subdir, with special exception for cp -R (see comments below).
+
+    Args:
+      exp_dst_uri: Wildcard-expanding dst_uri.
+      have_existing_dst_container: bool indicator of whether exp_dst_uri
+        names a container (directory, bucket, or existing bucket subdir).
+      command_name: Name of command making call. May not be the same as
+          self.command_name in the case of commands implemented atop other
+          commands (like mv command).
+
+    Raises:
+      CommandException: if the URI being checked does not name a container.
+    """
+    if exp_dst_uri.is_file_uri():
+      ok = exp_dst_uri.names_directory()
+    else:
+      if have_existing_dst_container:
+        ok = True
+      else:
+        # It's ok to specify a non-existing bucket subdir, for example:
+        #   gsutil cp -R dir gs://bucket/abc
+        # where gs://bucket/abc isn't an existing subdir.
+        ok = exp_dst_uri.names_object()
+    if not ok:
+      raise CommandException('Destination URI must name a directory, bucket, '
+                             'or bucket\nsubdirectory for the multiple '
+                             'source form of the %s command.' % command_name)
+
+  class _FileCopyCallbackHandler(object):
+    """Outputs progress info for large copy requests."""
+
+    def __init__(self, upload):
+      if upload:
+        self.announce_text = 'Uploading'
+      else:
+        self.announce_text = 'Downloading'
+
+    def call(self, total_bytes_transferred, total_size):
+      sys.stderr.write('%s: %s/%s    \r' % (
+          self.announce_text,
+          MakeHumanReadable(total_bytes_transferred),
+          MakeHumanReadable(total_size)))
+      if total_bytes_transferred == total_size:
+        sys.stderr.write('\n')
+
+  class _StreamCopyCallbackHandler(object):
+    """Outputs progress info for Stream copy to cloud.
+       Total Size of the stream is not known, so we output
+       only the bytes transferred.
+    """
+
+    def call(self, total_bytes_transferred, total_size):
+      sys.stderr.write('Uploading: %s    \r' % (
+          MakeHumanReadable(total_bytes_transferred)))
+      if total_size and total_bytes_transferred == total_size:
+        sys.stderr.write('\n')
+
+  def _GetTransferHandlers(self, dst_uri, size, upload):
+    """
+    Selects upload/download and callback handlers.
+
+    We use a callback handler that shows a simple textual progress indicator
+    if size is above the configurable threshold.
+
+    We use a resumable transfer handler if size is >= the configurable
+    threshold and resumable transfers are supported by the given provider.
+    boto supports resumable downloads for all providers, but resumable
+    uploads are currently only supported by GS.
+
+    Args:
+      dst_uri: the destination URI.
+      size: size of file (object) being uploaded (downloaded).
+      upload: bool indication of whether transfer is an upload.
+    """
+    config = boto.config
+    resumable_threshold = config.getint('GSUtil', 'resumable_threshold', TWO_MB)
+    transfer_handler = None
+    cb = None
+    num_cb = None
+
+    # Checks whether the destination file is a "special" file, like /dev/null on
+    # Linux platforms or null on Windows platforms, so we can disable resumable
+    # download support since the file size of the destination won't ever be
+    # correct.
+    dst_is_special = False
+    if dst_uri.is_file_uri():
+      # Check explicitly first because os.stat doesn't work on 'nul' in Windows.
+      if dst_uri.object_name == os.devnull:
+        dst_is_special = True
+      try:
+        mode = os.stat(dst_uri.object_name).st_mode
+        if stat.S_ISCHR(mode):
+          dst_is_special = True
+      except OSError:
+        pass
+
+    if size >= resumable_threshold and not dst_is_special:
+      if not self.quiet:
+        cb = self._FileCopyCallbackHandler(upload).call
+        num_cb = int(size / TWO_MB)
+
+      resumable_tracker_dir = config.get(
+          'GSUtil', 'resumable_tracker_dir',
+          os.path.expanduser('~' + os.sep + '.gsutil'))
+      if not os.path.exists(resumable_tracker_dir):
+        os.makedirs(resumable_tracker_dir)
+
+      if upload:
+        # Encode the dest bucket and object name into the tracker file name.
+        res_tracker_file_name = (
+            re.sub('[/\\\\]', '_', 'resumable_upload__%s__%s.url' %
+                   (dst_uri.bucket_name, dst_uri.object_name)))
+      else:
+        # Encode the fully-qualified dest file name into the tracker file name.
+        res_tracker_file_name = (
+            re.sub('[/\\\\]', '_', 'resumable_download__%s.etag' %
+                   (os.path.realpath(dst_uri.object_name))))
+
+      res_tracker_file_name = _hash_filename(res_tracker_file_name)
+      tracker_file = '%s%s%s' % (resumable_tracker_dir, os.sep,
+                                 res_tracker_file_name)
+      if upload:
+        if dst_uri.scheme == 'gs':
+          transfer_handler = ResumableUploadHandler(tracker_file)
+      else:
+        transfer_handler = ResumableDownloadHandler(tracker_file)
+
+    return (cb, num_cb, transfer_handler)
+
+  def _LogCopyOperation(self, src_uri, dst_uri, headers):
+    """
+    Logs copy operation being performed, including Content-Type if appropriate.
+    """
+    if self.quiet:
+      return
+    if 'Content-Type' in headers and dst_uri.is_cloud_uri():
+      content_type_msg = ' [Content-Type=%s]' % headers['Content-Type']
+    else:
+      content_type_msg = ''
+    if src_uri.is_stream():
+      self.THREADED_LOGGER.info('Copying from <STDIN>%s...', content_type_msg)
+    else:
+      self.THREADED_LOGGER.info('Copying %s%s...', src_uri, content_type_msg)
+
+  # We pass the headers explicitly to this call instead of using self.headers
+  # so we can set different metadata (like Content-Type type) for each object.
+  def _CopyObjToObjInTheCloud(self, src_key, src_uri, dst_uri, headers):
+    """Performs copy-in-the cloud from specified src to dest object.
+
+    Args:
+      src_key: Source Key.
+      src_uri: Source StorageUri.
+      dst_uri: Destination StorageUri.
+      headers: A copy of the headers dictionary.
+
+    Returns:
+      (elapsed_time, bytes_transferred, dst_uri) excluding overhead like initial
+      HEAD. Note: At present copy-in-the-cloud doesn't return the generation of
+      the created object, so the returned URI is actually not version-specific
+      (unlike other cp cases).
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    self._SetContentTypeHeader(src_uri, headers)
+    self._LogCopyOperation(src_uri, dst_uri, headers)
+    # Do Object -> object copy within same provider (uses
+    # x-<provider>-copy-source metadata HTTP header to request copying at the
+    # server).
+    src_bucket = src_uri.get_bucket(False, headers)
+    preserve_acl = False
+    canned_acl = None
+    if self.sub_opts:
+      for o, a in self.sub_opts:
+        if o == '-a':
+          canned_acls = dst_uri.canned_acls()
+          if a not in canned_acls:
+            raise CommandException('Invalid canned ACL "%s".' % a)
+          canned_acl = a
+          headers[dst_uri.get_provider().acl_header] = canned_acl
+        if o == '-p':
+          preserve_acl = True
+    if preserve_acl and canned_acl:
+      raise CommandException(
+          'Specifying both the -p and -a options together is invalid.')
+    start_time = time.time()
+    # Pass headers in headers param not metadata param, so boto will copy
+    # existing key's metadata and just set the additional headers specified
+    # in the headers param (rather than using the headers to override existing
+    # metadata). In particular this allows us to copy the existing key's
+    # Content-Type and other metadata users need while still being able to
+    # set headers the API needs (like x-goog-project-id). Note that this means
+    # you can't do something like:
+    #   gsutil cp -t Content-Type text/html gs://bucket/* gs://bucket2
+    # to change the Content-Type while copying.
+
+    try:
+      dst_key = dst_uri.copy_key(
+          src_bucket.name, src_uri.object_name, preserve_acl=False,
+          headers=headers, src_version_id=src_uri.version_id,
+          src_generation=src_uri.generation)
+    except GSResponseError as e:
+      exc_name, error_detail = ExtractErrorDetail(e)
+      if (exc_name == 'GSResponseError'
+          and ('Copy-in-the-cloud disallowed' in error_detail)):
+          raise CommandException('%s.\nNote: you can copy between locations '
+                                 'and between storage classes by using the '
+                                 'gsutil cp -D option.' % error_detail)
+      else:
+        raise
+    end_time = time.time()
+    return (end_time - start_time, src_key.size,
+            dst_uri.clone_replace_key(dst_key))
+
+  def _CheckFreeSpace(self, path):
+    """Return path/drive free space (in bytes)."""
+    if platform.system() == 'Windows':
+      from ctypes import c_int, c_uint64, c_wchar_p, windll, POINTER, WINFUNCTYPE, WinError
+      try:
+        GetDiskFreeSpaceEx = WINFUNCTYPE(c_int, c_wchar_p, POINTER(c_uint64),
+                                         POINTER(c_uint64), POINTER(c_uint64))
+        GetDiskFreeSpaceEx = GetDiskFreeSpaceEx(
+            ('GetDiskFreeSpaceExW', windll.kernel32), (
+                (1, 'lpszPathName'),
+                (2, 'lpFreeUserSpace'),
+                (2, 'lpTotalSpace'),
+                (2, 'lpFreeSpace'),))
+      except AttributeError:
+        GetDiskFreeSpaceEx = WINFUNCTYPE(c_int, c_char_p, POINTER(c_uint64),
+                                         POINTER(c_uint64), POINTER(c_uint64))
+        GetDiskFreeSpaceEx = GetDiskFreeSpaceEx(
+            ('GetDiskFreeSpaceExA', windll.kernel32), (
+                (1, 'lpszPathName'),
+                (2, 'lpFreeUserSpace'),
+                (2, 'lpTotalSpace'),
+                (2, 'lpFreeSpace'),))
+
+      def GetDiskFreeSpaceEx_errcheck(result, func, args):
+        if not result:
+            raise WinError()
+        return args[1].value
+      GetDiskFreeSpaceEx.errcheck = GetDiskFreeSpaceEx_errcheck
+
+      return GetDiskFreeSpaceEx(os.getenv('SystemDrive'))
+    else:
+      (_, f_frsize, _, _, f_bavail, _, _, _, _, _) = os.statvfs(path)
+      return f_frsize * f_bavail
+
+  def _PerformResumableUploadIfApplies(self, fp, dst_uri, canned_acl, headers):
+    """
+    Performs resumable upload if supported by provider and file is above
+    threshold, else performs non-resumable upload.
+
+    Returns (elapsed_time, bytes_transferred, version-specific dst_uri).
+    """
+    start_time = time.time()
+    # Determine file size different ways for case where fp is actually a wrapper
+    # around a Key vs an actual file.
+    if isinstance(fp, KeyFile):
+      file_size = fp.getkey().size
+    else:
+      file_size = os.path.getsize(fp.name)
+    (cb, num_cb, res_upload_handler) = self._GetTransferHandlers(
+        dst_uri, file_size, True)
+    if dst_uri.scheme == 'gs':
+      # Resumable upload protocol is Google Cloud Storage-specific.
+      dst_uri.set_contents_from_file(fp, headers, policy=canned_acl,
+                                     cb=cb, num_cb=num_cb,
+                                     res_upload_handler=res_upload_handler)
+    else:
+      dst_uri.set_contents_from_file(fp, headers, policy=canned_acl,
+                                     cb=cb, num_cb=num_cb)
+    if res_upload_handler:
+      # ResumableUploadHandler does not update upload_start_point from its
+      # initial value of -1 if transferring the whole file, so clamp at 0
+      bytes_transferred = file_size - max(
+                              res_upload_handler.upload_start_point, 0)
+    else:
+      bytes_transferred = file_size
+    end_time = time.time()
+    return (end_time - start_time, bytes_transferred, dst_uri)
+
+  def _PerformStreamingUpload(self, fp, dst_uri, headers, canned_acl=None):
+    """
+    Performs a streaming upload to the cloud.
+
+    Args:
+      fp: The file whose contents to upload.
+      dst_uri: Destination StorageUri.
+      headers: A copy of the headers dictionary.
+      canned_acl: Optional canned ACL to set on the object.
+
+    Returns (elapsed_time, bytes_transferred, version-specific dst_uri).
+    """
+    start_time = time.time()
+
+    if self.quiet:
+      cb = None
+    else:
+      cb = self._StreamCopyCallbackHandler().call
+    dst_uri.set_contents_from_stream(
+        fp, headers, policy=canned_acl, cb=cb)
+    try:
+      bytes_transferred = fp.tell()
+    except:
+      bytes_transferred = 0
+
+    end_time = time.time()
+    return (end_time - start_time, bytes_transferred, dst_uri)
+
+  def _SetContentTypeHeader(self, src_uri, headers):
+    """
+    Sets content type header to value specified in '-h Content-Type' option (if
+    specified); else sets using Content-Type detection.
+    """
+    if 'Content-Type' in headers:
+      # If empty string specified (i.e., -h "Content-Type:") set header to None,
+      # which will inhibit boto from sending the CT header. Otherwise, boto will
+      # pass through the user specified CT header.
+      if not headers['Content-Type']:
+        headers['Content-Type'] = None
+      # else we'll keep the value passed in via -h option (not performing
+      # content type detection).
+    else:
+      # Only do content type recognition is src_uri is a file. Object-to-object
+      # copies with no -h Content-Type specified re-use the content type of the
+      # source object.
+      if src_uri.is_file_uri():
+        object_name = src_uri.object_name
+        content_type = None
+        # Streams (denoted by '-') are expected to be 'application/octet-stream'
+        # and 'file' would partially consume them.
+        if object_name != '-':
+          if self.USE_MAGICFILE:
+            p = subprocess.Popen(['file', '--mime-type', object_name],
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            output, error = p.communicate()
+            if p.returncode != 0 or error:
+              raise CommandException(
+                  'Encountered error running "file --mime-type %s" '
+                  '(returncode=%d).\n%s' % (object_name, p.returncode, error))
+            # Parse output by removing line delimiter and splitting on last ":
+            content_type = output.rstrip().rpartition(': ')[2]
+          else:
+            content_type = mimetypes.guess_type(object_name)[0]
+        if not content_type:
+          content_type = self.DEFAULT_CONTENT_TYPE
+        headers['Content-Type'] = content_type
+
+  def _UploadFileToObject(self, src_key, src_uri, dst_uri, headers,
+                          should_log=True):
+    """Uploads a local file to an object.
+
+    Args:
+      src_key: Source StorageUri. Must be a file URI.
+      src_uri: Source StorageUri.
+      dst_uri: Destination StorageUri.
+      headers: The headers dictionary.
+      should_log: bool indicator whether we should log this operation.
+    Returns:
+      (elapsed_time, bytes_transferred, version-specific dst_uri), excluding
+      overhead like initial HEAD.
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    gzip_exts = []
+    canned_acl = None
+    if self.sub_opts:
+      for o, a in self.sub_opts:
+        if o == '-a':
+          canned_acls = dst_uri.canned_acls()
+          if a not in canned_acls:
+            raise CommandException('Invalid canned ACL "%s".' % a)
+          canned_acl = a
+        elif o == '-t':
+          print('Warning: -t is deprecated, and will be removed in the future. '
+                'Content type\ndetection is '
+                'now performed by default, unless inhibited by specifying '
+                'a\nContent-Type header via the -h option.')
+        elif o == '-z':
+          gzip_exts = a.split(',')
+
+    self._SetContentTypeHeader(src_uri, headers)
+    if should_log:
+      self._LogCopyOperation(src_uri, dst_uri, headers)
+
+    if 'Content-Language' not in headers:
+       content_language = config.get_value('GSUtil', 'content_language')
+       if content_language:
+         headers['Content-Language'] = content_language
+
+    fname_parts = src_uri.object_name.split('.')
+    if len(fname_parts) > 1 and fname_parts[-1] in gzip_exts:
+      if self.debug:
+        print 'Compressing %s (to tmp)...' % src_key
+      (gzip_fh, gzip_path) = tempfile.mkstemp()
+      gzip_fp = None
+      try:
+        # Check for temp space. Assume the compressed object is at most 2x
+        # the size of the object (normally should compress to smaller than
+        # the object)
+        if (self._CheckFreeSpace(gzip_path)
+            < 2*int(os.path.getsize(src_key.name))):
+          raise CommandException('Inadequate temp space available to compress '
+                                 '%s' % src_key.name)
+        gzip_fp = gzip.open(gzip_path, 'wb')
+        gzip_fp.writelines(src_key.fp)
+      finally:
+        if gzip_fp:
+          gzip_fp.close()
+        os.close(gzip_fh)
+      headers['Content-Encoding'] = 'gzip'
+      gzip_fp = open(gzip_path, 'rb')
+      try:
+        (elapsed_time, bytes_transferred, result_uri) = (
+            self._PerformResumableUploadIfApplies(gzip_fp, dst_uri,
+                                                  canned_acl, headers))
+      finally:
+        gzip_fp.close()
+      try:
+        os.unlink(gzip_path)
+      # Windows sometimes complains the temp file is locked when you try to
+      # delete it.
+      except Exception, e:
+        pass
+    elif (src_key.is_stream()
+          and dst_uri.get_provider().supports_chunked_transfer()):
+      (elapsed_time, bytes_transferred, result_uri) = (
+          self._PerformStreamingUpload(src_key.fp, dst_uri, headers,
+                                       canned_acl))
+    else:
+      if src_key.is_stream():
+        # For Providers that doesn't support chunked Transfers
+        tmp = tempfile.NamedTemporaryFile()
+        file_uri = self.suri_builder.StorageUri('file://%s' % tmp.name)
+        try:
+          file_uri.new_key(False, headers).set_contents_from_file(
+              src_key.fp, headers)
+          src_key = file_uri.get_key()
+        finally:
+          file_uri.close()
+      try:
+        (elapsed_time, bytes_transferred, result_uri) = (
+            self._PerformResumableUploadIfApplies(src_key.fp, dst_uri,
+                                                  canned_acl, headers))
+      finally:
+        if src_key.is_stream():
+          tmp.close()
+        else:
+          src_key.close()
+
+    return (elapsed_time, bytes_transferred, result_uri)
+
+  def _DownloadObjectToFile(self, src_key, src_uri, dst_uri, headers,
+                            should_log=True):
+    """Downloads an object to a local file.
+
+    Args:
+      src_key: Source StorageUri. Must be a file URI.
+      src_uri: Source StorageUri.
+      dst_uri: Destination StorageUri.
+      headers: The headers dictionary.
+      should_log: bool indicator whether we should log this operation.
+    Returns:
+      (elapsed_time, bytes_transferred, dst_uri), excluding overhead like
+      initial HEAD.
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    if should_log:
+      self._LogCopyOperation(src_uri, dst_uri, headers)
+    (cb, num_cb, res_download_handler) = self._GetTransferHandlers(
+        dst_uri, src_key.size, False)
+    file_name = dst_uri.object_name
+    dir_name = os.path.dirname(file_name)
+    if dir_name and not os.path.exists(dir_name):
+      # Do dir creation in try block so can ignore case where dir already
+      # exists. This is needed to avoid a race condition when running gsutil
+      # -m cp.
+      try:
+        os.makedirs(dir_name)
+      except OSError, e:
+        if e.errno != errno.EEXIST:
+          raise
+    # For gzipped objects not named *.gz download to a temp file and unzip.
+    if (hasattr(src_key, 'content_encoding')
+        and src_key.content_encoding == 'gzip'
+        and not file_name.endswith('.gz')):
+      # We can't use tempfile.mkstemp() here because we need a predictable
+      # filename for resumable downloads.
+      download_file_name = '%s_.gztmp' % file_name
+      need_to_unzip = True
+    else:
+      download_file_name = file_name
+      need_to_unzip = False
+    fp = None
+    try:
+      if res_download_handler:
+        fp = open(download_file_name, 'ab')
+      else:
+        fp = open(download_file_name, 'wb')
+      start_time = time.time()
+      src_key.get_contents_to_file(fp, headers, cb=cb, num_cb=num_cb,
+                                   res_download_handler=res_download_handler)
+      # If a custom test method is defined, call it here. For the copy command,
+      # test methods are expected to take one argument: an open file pointer,
+      # and are used to perturb the open file during download to exercise
+      # download error detection.
+      if self.test_method:
+        self.test_method(fp)
+      end_time = time.time()
+    finally:
+      if fp:
+        fp.close()
+
+    # Discard the md5 if we are resuming a partial download.
+    if res_download_handler and res_download_handler.download_start_point:
+      src_key.md5 = None
+
+    # Verify downloaded file checksum matched source object's checksum.
+    self._CheckFinalMd5(src_key, download_file_name)
+
+    if res_download_handler:
+      bytes_transferred = (
+          src_key.size - res_download_handler.download_start_point)
+    else:
+      bytes_transferred = src_key.size
+    if need_to_unzip:
+      # Log that we're uncompressing if the file is big enough that
+      # decompressing would make it look like the transfer "stalled" at the end.
+      if not self.quiet and bytes_transferred > 10 * 1024 * 1024:
+        self.THREADED_LOGGER.info('Uncompressing downloaded tmp file to %s...',
+                                  file_name)
+      # Downloaded gzipped file to a filename w/o .gz extension, so unzip.
+      f_in = gzip.open(download_file_name, 'rb')
+      f_out = open(file_name, 'wb')
+      try:
+        while True:
+          data = f_in.read(8192)
+          if not data:
+            break
+          f_out.write(data)
+      finally:
+        f_out.close()
+        f_in.close()
+        os.unlink(download_file_name)
+    return (end_time - start_time, bytes_transferred, dst_uri)
+
+  def _PerformDownloadToStream(self, src_key, src_uri, str_fp, headers):
+    (cb, num_cb, res_download_handler) = self._GetTransferHandlers(
+                                src_uri, src_key.size, False)
+    start_time = time.time()
+    src_key.get_contents_to_file(str_fp, headers, cb=cb, num_cb=num_cb)
+    end_time = time.time()
+    bytes_transferred = src_key.size
+    end_time = time.time()
+    return (end_time - start_time, bytes_transferred)
+
+  def _CopyFileToFile(self, src_key, src_uri, dst_uri, headers):
+    """Copies a local file to a local file.
+
+    Args:
+      src_key: Source StorageUri. Must be a file URI.
+      src_uri: Source StorageUri.
+      dst_uri: Destination StorageUri.
+      headers: The headers dictionary.
+    Returns:
+      (elapsed_time, bytes_transferred, dst_uri), excluding
+      overhead like initial HEAD.
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    self._LogCopyOperation(src_uri, dst_uri, headers)
+    dst_key = dst_uri.new_key(False, headers)
+    start_time = time.time()
+    dst_key.set_contents_from_file(src_key.fp, headers)
+    end_time = time.time()
+    return (end_time - start_time, os.path.getsize(src_key.fp.name), dst_uri)
+
+  def _CopyObjToObjDaisyChainMode(self, src_key, src_uri, dst_uri, headers):
+    """Copies from src_uri to dst_uri in "daisy chain" mode.
+       See -D OPTION documentation about what daisy chain mode is.
+
+    Args:
+      src_key: Source Key.
+      src_uri: Source StorageUri.
+      dst_uri: Destination StorageUri.
+      headers: A copy of the headers dictionary.
+
+    Returns:
+      (elapsed_time, bytes_transferred, version-specific dst_uri) excluding
+      overhead like initial HEAD.
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    self._SetContentTypeHeader(src_uri, headers)
+    self._LogCopyOperation(src_uri, dst_uri, headers)
+    canned_acl = None
+    if self.sub_opts:
+      for o, a in self.sub_opts:
+        if o == '-a':
+          canned_acls = dst_uri.canned_acls()
+          if a not in canned_acls:
+            raise CommandException('Invalid canned ACL "%s".' % a)
+          canned_acl = a
+        elif o == '-p':
+          # We don't attempt to preserve ACLs across providers because
+          # GCS and S3 support different ACLs and disjoint principals.
+          raise NotImplementedError('Cross-provider cp -p not supported')
+    return self._PerformResumableUploadIfApplies(KeyFile(src_key), dst_uri,
+                                                 canned_acl, headers)
+
+  def _PerformCopy(self, src_uri, dst_uri):
+    """Performs copy from src_uri to dst_uri, handling various special cases.
+
+    Args:
+      src_uri: Source StorageUri.
+      dst_uri: Destination StorageUri.
+
+    Returns:
+      (elapsed_time, bytes_transferred, version-specific dst_uri) excluding
+      overhead like initial HEAD.
+
+    Raises:
+      CommandException: if errors encountered.
+    """
+    # Make a copy of the input headers each time so we can set a different
+    # content type for each object.
+    if self.headers:
+      headers = self.headers.copy()
+    else:
+      headers = {}
+
+    src_key = src_uri.get_key(False, headers)
+    if not src_key:
+      raise CommandException('"%s" does not exist.' % src_uri)
+
+    # On Windows, stdin is opened as text mode instead of binary which causes
+    # problems when piping a binary file, so this switches it to binary mode.
+    if IS_WINDOWS and src_uri.is_file_uri() and src_key.is_stream():
+      import msvcrt
+      msvcrt.setmode(src_key.fp.fileno(), os.O_BINARY)
+
+    if self.no_clobber:
+        # There are two checks to prevent clobbering:
+        # 1) The first check is to see if the item
+        #    already exists at the destination and prevent the upload/download
+        #    from happening. This is done by the exists() call.
+        # 2) The second check is only relevant if we are writing to gs. We can
+        #    enforce that the server only writes the object if it doesn't exist
+        #    by specifying the header below. This check only happens at the
+        #    server after the complete file has been uploaded. We specify this
+        #    header to prevent a race condition where a destination file may
+        #    be created after the first check and before the file is fully
+        #    uploaded.
+        # In order to save on unnecessary uploads/downloads we perform both
+        # checks. However, this may come at the cost of additional HTTP calls.
+        if dst_uri.exists(headers):
+          if not self.quiet:
+            self.THREADED_LOGGER.info('Skipping existing item: %s' %
+                                      dst_uri.uri)
+          return (0, 0, None)
+        if dst_uri.is_cloud_uri() and dst_uri.scheme == 'gs':
+          headers['x-goog-if-generation-match'] = '0'
+
+    if src_uri.is_cloud_uri() and dst_uri.is_cloud_uri():
+      if src_uri.scheme == dst_uri.scheme and not self.daisy_chain:
+        return self._CopyObjToObjInTheCloud(src_key, src_uri, dst_uri, headers)
+      else:
+        return self._CopyObjToObjDaisyChainMode(src_key, src_uri, dst_uri,
+                                                headers)
+    elif src_uri.is_file_uri() and dst_uri.is_cloud_uri():
+      return self._UploadFileToObject(src_key, src_uri, dst_uri, headers)
+    elif src_uri.is_cloud_uri() and dst_uri.is_file_uri():
+      return self._DownloadObjectToFile(src_key, src_uri, dst_uri, headers)
+    elif src_uri.is_file_uri() and dst_uri.is_file_uri():
+      return self._CopyFileToFile(src_key, src_uri, dst_uri, headers)
+    else:
+      raise CommandException('Unexpected src/dest case')
+
+  def _ExpandDstUri(self, dst_uri_str):
+    """
+    Expands wildcard if present in dst_uri_str.
+
+    Args:
+      dst_uri_str: String representation of requested dst_uri.
+
+    Returns:
+        (exp_dst_uri, have_existing_dst_container)
+        where have_existing_dst_container is a bool indicating whether
+        exp_dst_uri names an existing directory, bucket, or bucket subdirectory.
+
+    Raises:
+      CommandException: if dst_uri_str matched more than 1 URI.
+    """
+    dst_uri = self.suri_builder.StorageUri(dst_uri_str)
+
+    # Handle wildcarded dst_uri case.
+    if ContainsWildcard(dst_uri):
+      blr_expansion = list(self.WildcardIterator(dst_uri))
+      if len(blr_expansion) != 1:
+        raise CommandException('Destination (%s) must match exactly 1 URI' %
+                               dst_uri_str)
+      blr = blr_expansion[0]
+      uri = blr.GetUri()
+      if uri.is_cloud_uri():
+        return (uri, uri.names_bucket() or blr.HasPrefix()
+                or blr.GetKey().endswith('/'))
+      else:
+        return (uri, uri.names_directory())
+
+    # Handle non-wildcarded dst_uri:
+    if dst_uri.is_file_uri():
+      return (dst_uri, dst_uri.names_directory())
+    if dst_uri.names_bucket():
+      return (dst_uri, True)
+    # For object URIs check 3 cases: (a) if the name ends with '/' treat as a
+    # subdir; else, perform a wildcard expansion with dst_uri + "*" and then
+    # find if (b) there's a Prefix matching dst_uri, or (c) name is of form
+    # dir_$folder$ (and in both these cases also treat dir as a subdir).
+    if dst_uri.is_cloud_uri() and dst_uri_str.endswith('/'):
+      return (dst_uri, True)
+    blr_expansion = list(self.WildcardIterator(
+        '%s*' % dst_uri_str.rstrip(dst_uri.delim)))
+    for blr in blr_expansion:
+      if blr.GetRStrippedUriString().endswith('_$folder$'):
+        return (dst_uri, True)
+      if blr.GetRStrippedUriString() == dst_uri_str.rstrip(dst_uri.delim):
+        return (dst_uri, blr.HasPrefix())
+    return (dst_uri, False)
+
+  def _ConstructDstUri(self, src_uri, exp_src_uri,
+                       src_uri_names_container, src_uri_expands_to_multi,
+                       have_multiple_srcs, exp_dst_uri,
+                       have_existing_dest_subdir):
+    """
+    Constructs the destination URI for a given exp_src_uri/exp_dst_uri pair,
+    using context-dependent naming rules that mimic Linux cp and mv behavior.
+
+    Args:
+      src_uri: src_uri to be copied.
+      exp_src_uri: Single StorageUri from wildcard expansion of src_uri.
+      src_uri_names_container: True if src_uri names a container (including the
+          case of a wildcard-named bucket subdir (like gs://bucket/abc,
+          where gs://bucket/abc/* matched some objects). Note that this is
+          additional semantics tha src_uri.names_container() doesn't understand
+          because the latter only understands StorageUris, not wildcards.
+      src_uri_expands_to_multi: True if src_uri expanded to multiple URIs.
+      have_multiple_srcs: True if this is a multi-source request. This can be
+          true if src_uri wildcard-expanded to multiple URIs or if there were
+          multiple source URIs in the request.
+      exp_dst_uri: the expanded StorageUri requested for the cp destination.
+          Final written path is constructed from this plus a context-dependent
+          variant of src_uri.
+      have_existing_dest_subdir: bool indicator whether dest is an existing
+        subdirectory.
+
+    Returns:
+      StorageUri to use for copy.
+
+    Raises:
+      CommandException if destination object name not specified for
+      source and source is a stream.
+    """
+    if self._ShouldTreatDstUriAsSingleton(
+        have_multiple_srcs, have_existing_dest_subdir, exp_dst_uri):
+      # We're copying one file or object to one file or object.
+      return exp_dst_uri
+
+    if exp_src_uri.is_stream():
+      if exp_dst_uri.names_container():
+        raise CommandException('Destination object name needed when '
+                               'source is a stream')
+      return exp_dst_uri
+
+    if not self.recursion_requested and not have_multiple_srcs:
+      # We're copying one file or object to a subdirectory. Append final comp
+      # of exp_src_uri to exp_dst_uri.
+      src_final_comp = exp_src_uri.object_name.rpartition(src_uri.delim)[-1]
+      return self.suri_builder.StorageUri('%s%s%s' % (
+          exp_dst_uri.uri.rstrip(exp_dst_uri.delim), exp_dst_uri.delim,
+          src_final_comp))
+
+    # Else we're copying multiple sources to a directory, bucket, or a bucket
+    # "sub-directory".
+
+    # Ensure exp_dst_uri ends in delim char if we're doing a multi-src copy or
+    # a copy to a directory. (The check for copying to a directory needs
+    # special-case handling so that the command:
+    #   gsutil cp gs://bucket/obj dir
+    # will turn into file://dir/ instead of file://dir -- the latter would cause
+    # the file "dirobj" to be created.)
+    # Note: need to check have_multiple_srcs or src_uri.names_container()
+    # because src_uri could be a bucket containing a single object, named
+    # as gs://bucket.
+    if ((have_multiple_srcs or src_uri.names_container()
+         or os.path.isdir(exp_dst_uri.object_name))
+        and not exp_dst_uri.uri.endswith(exp_dst_uri.delim)):
+      exp_dst_uri = exp_dst_uri.clone_replace_name(
+         '%s%s' % (exp_dst_uri.object_name, exp_dst_uri.delim)
+      )
+
+    # Making naming behavior match how things work with local Linux cp and mv
+    # operations depends on many factors, including whether the destination is a
+    # container, the plurality of the source(s), and whether the mv command is
+    # being used:
+    # 1. For the "mv" command that specifies a non-existent destination subdir,
+    #    renaming should occur at the level of the src subdir, vs appending that
+    #    subdir beneath the dst subdir like is done for copying. For example:
+    #      gsutil rm -R gs://bucket
+    #      gsutil cp -R dir1 gs://bucket
+    #      gsutil cp -R dir2 gs://bucket/subdir1
+    #      gsutil mv gs://bucket/subdir1 gs://bucket/subdir2
+    #    would (if using cp naming behavior) end up with paths like:
+    #      gs://bucket/subdir2/subdir1/dir2/.svn/all-wcprops
+    #    whereas mv naming behavior should result in:
+    #      gs://bucket/subdir2/dir2/.svn/all-wcprops
+    # 2. Copying from directories, buckets, or bucket subdirs should result in
+    #    objects/files mirroring the source directory hierarchy. For example:
+    #      gsutil cp dir1/dir2 gs://bucket
+    #    should create the object gs://bucket/dir2/file2, assuming dir1/dir2
+    #    contains file2).
+    #    To be consistent with Linux cp behavior, there's one more wrinkle when
+    #    working with subdirs: The resulting object names depend on whether the
+    #    destination subdirectory exists. For example, if gs://bucket/subdir
+    #    exists, the command:
+    #      gsutil cp -R dir1/dir2 gs://bucket/subdir
+    #    should create objects named like gs://bucket/subdir/dir2/a/b/c. In
+    #    contrast, if gs://bucket/subdir does not exist, this same command
+    #    should create objects named like gs://bucket/subdir/a/b/c.
+    # 3. Copying individual files or objects to dirs, buckets or bucket subdirs
+    #    should result in objects/files named by the final source file name
+    #    component. Example:
+    #      gsutil cp dir1/*.txt gs://bucket
+    #    should create the objects gs://bucket/f1.txt and gs://bucket/f2.txt,
+    #    assuming dir1 contains f1.txt and f2.txt.
+
+    if (self.perform_mv and self.recursion_requested
+        and src_uri_expands_to_multi and not have_existing_dest_subdir):
+      # Case 1. Handle naming rules for bucket subdir mv. Here we want to
+      # line up the src_uri against its expansion, to find the base to build
+      # the new name. For example, running the command:
+      #   gsutil mv gs://bucket/abcd gs://bucket/xyz
+      # when processing exp_src_uri=gs://bucket/abcd/123
+      # exp_src_uri_tail should become /123
+      # Note: mv.py code disallows wildcard specification of source URI.
+      exp_src_uri_tail = exp_src_uri.uri[len(src_uri.uri):]
+      dst_key_name = '%s/%s' % (exp_dst_uri.object_name.rstrip('/'),
+                                exp_src_uri_tail.strip('/'))
+      return exp_dst_uri.clone_replace_name(dst_key_name)
+
+    if src_uri_names_container and not exp_dst_uri.names_file():
+      # Case 2. Build dst_key_name from subpath of exp_src_uri past
+      # where src_uri ends. For example, for src_uri=gs://bucket/ and
+      # exp_src_uri=gs://bucket/src_subdir/obj, dst_key_name should be
+      # src_subdir/obj.
+      src_uri_path_sans_final_dir = _GetPathBeforeFinalDir(src_uri)
+      dst_key_name = exp_src_uri.uri[
+         len(src_uri_path_sans_final_dir):].lstrip(src_uri.delim)
+      # Handle case where dst_uri is a non-existent subdir.
+      if not have_existing_dest_subdir:
+        dst_key_name = dst_key_name.partition(src_uri.delim)[-1]
+      # Handle special case where src_uri was a directory named with '.' or
+      # './', so that running a command like:
+      #   gsutil cp -r . gs://dest
+      # will produce obj names of the form gs://dest/abc instead of
+      # gs://dest/./abc.
+      if dst_key_name.startswith('.%s' % os.sep):
+        dst_key_name = dst_key_name[2:]
+
+    else:
+      # Case 3.
+      dst_key_name = exp_src_uri.object_name.rpartition(src_uri.delim)[-1]
+
+    if (exp_dst_uri.is_file_uri()
+        or self._ShouldTreatDstUriAsBucketSubDir(
+            have_multiple_srcs, exp_dst_uri, have_existing_dest_subdir)):
+      if exp_dst_uri.object_name.endswith(exp_dst_uri.delim):
+        dst_key_name = '%s%s%s' % (
+            exp_dst_uri.object_name.rstrip(exp_dst_uri.delim),
+            exp_dst_uri.delim, dst_key_name)
+      else:
+        delim = exp_dst_uri.delim if exp_dst_uri.object_name else ''
+        dst_key_name = '%s%s%s' % (exp_dst_uri.object_name, delim, dst_key_name)
+
+    return exp_dst_uri.clone_replace_name(dst_key_name)
+
+  def _FixWindowsNaming(self, src_uri, dst_uri):
+    """
+    Rewrites the destination URI built by _ConstructDstUri() to translate
+    Windows pathnames to cloud pathnames if needed.
+
+    Args:
+      src_uri: Source URI to be copied.
+      dst_uri: The destination URI built by _ConstructDstUri().
+
+    Returns:
+      StorageUri to use for copy.
+    """
+    if (src_uri.is_file_uri() and src_uri.delim == '\\'
+        and dst_uri.is_cloud_uri()):
+      trans_uri_str = re.sub(r'\\', '/', dst_uri.uri)
+      dst_uri = self.suri_builder.StorageUri(trans_uri_str)
+    return dst_uri
+
+  # Command entry point.
+  def RunCommand(self):
+
+    # Inner funcs.
+    def _CopyExceptionHandler(e):
+      """Simple exception handler to allow post-completion status."""
+      self.THREADED_LOGGER.error(str(e))
+      self.copy_failure_count += 1
+
+    def _CopyFunc(name_expansion_result):
+      """Worker function for performing the actual copy (and rm, for mv)."""
+      if self.perform_mv:
+        cmd_name = 'mv'
+      else:
+        cmd_name = self.command_name
+      src_uri = self.suri_builder.StorageUri(
+          name_expansion_result.GetSrcUriStr())
+      exp_src_uri = self.suri_builder.StorageUri(
+          name_expansion_result.GetExpandedUriStr())
+      src_uri_names_container = name_expansion_result.NamesContainer()
+      src_uri_expands_to_multi = name_expansion_result.NamesContainer()
+      have_multiple_srcs = name_expansion_result.IsMultiSrcRequest()
+      have_existing_dest_subdir = (
+          name_expansion_result.HaveExistingDstContainer())
+
+      if src_uri.names_provider():
+        raise CommandException(
+            'The %s command does not allow provider-only source URIs (%s)' %
+            (cmd_name, src_uri))
+      if have_multiple_srcs:
+        self._InsistDstUriNamesContainer(exp_dst_uri,
+                                         have_existing_dst_container,
+                                         cmd_name)
+
+      if self.perform_mv:
+        if name_expansion_result.NamesContainer():
+          # Use recursion_requested when performing name expansion for the
+          # directory mv case so we can determine if any of the source URIs are
+          # directories (and then use cp -R and rm -R to perform the move, to
+          # match the behavior of Linux mv (which when moving a directory moves
+          # all the contained files).
+          self.recursion_requested = True
+          # Disallow wildcard src URIs when moving directories, as supporting it
+          # would make the name transformation too complex and would also be
+          # dangerous (e.g., someone could accidentally move many objects to the
+          # wrong name, or accidentally overwrite many objects).
+          if ContainsWildcard(src_uri):
+            raise CommandException('The mv command disallows naming source '
+                                   'directories using wildcards')
+
+      if (exp_dst_uri.is_file_uri()
+          and not os.path.exists(exp_dst_uri.object_name)
+          and have_multiple_srcs):
+        os.makedirs(exp_dst_uri.object_name)
+
+      dst_uri = self._ConstructDstUri(src_uri, exp_src_uri,
+                                      src_uri_names_container,
+                                      src_uri_expands_to_multi,
+                                      have_multiple_srcs, exp_dst_uri,
+                                      have_existing_dest_subdir)
+      dst_uri = self._FixWindowsNaming(src_uri, dst_uri)
+
+      self._CheckForDirFileConflict(exp_src_uri, dst_uri)
+      if self._SrcDstSame(exp_src_uri, dst_uri):
+        raise CommandException('%s: "%s" and "%s" are the same file - '
+                               'abort.' % (cmd_name, exp_src_uri, dst_uri))
+
+      if dst_uri.is_cloud_uri() and dst_uri.is_version_specific:
+        raise CommandException('%s: a version-specific URI\n(%s)\ncannot be '
+                               'the destination for gsutil cp - abort.'
+                               % (cmd_name, dst_uri))
+
+      elapsed_time = bytes_transferred = 0
+      try:
+        (elapsed_time, bytes_transferred, result_uri) = (
+            self._PerformCopy(exp_src_uri, dst_uri))
+      except Exception, e:
+        if self._IsNoClobberServerException(e):
+          if not self.quiet:
+            self.THREADED_LOGGER.info('Rejected (noclobber): %s' % dst_uri.uri)
+        elif self.continue_on_error:
+          if not self.quiet:
+            self.THREADED_LOGGER.error('Error copying %s: %s' % (src_uri.uri,
+              str(e)))
+          self.copy_failure_count += 1
+        else:
+          raise
+      if self.print_ver:
+        # Some cases don't return a version-specific URI (e.g., if destination
+        # is a file).
+        if hasattr(result_uri, 'version_specific_uri'):
+          self.THREADED_LOGGER.info('Created: %s' %
+                                    result_uri.version_specific_uri)
+        else:
+          self.THREADED_LOGGER.info('Created: %s' % result_uri.uri)
+
+      # TODO: If we ever use -n (noclobber) with -M (move) (not possible today
+      # since we call copy internally from move and don't specify the -n flag)
+      # we'll need to only remove the source when we have not skipped the
+      # destination.
+      if self.perform_mv:
+        if not self.quiet:
+          self.THREADED_LOGGER.info('Removing %s...', exp_src_uri)
+        exp_src_uri.delete_key(validate=False, headers=self.headers)
+      stats_lock.acquire()
+      self.total_elapsed_time += elapsed_time
+      self.total_bytes_transferred += bytes_transferred
+      stats_lock.release()
+
+    # Start of RunCommand code.
+    self._ParseArgs()
+
+    self.total_elapsed_time = self.total_bytes_transferred = 0
+    if self.args[-1] == '-' or self.args[-1] == 'file://-':
+      self._HandleStreamingDownload()
+      return 0
+
+    if self.read_args_from_stdin:
+      if len(self.args) != 1:
+        raise CommandException('Source URIs cannot be specified with -I option')
+      uri_strs = self._StdinIterator()
+    else:
+      if len(self.args) < 2:
+        raise CommandException('Wrong number of arguments for "cp" command.')
+      uri_strs = self.args[0:len(self.args)-1]
+
+    (exp_dst_uri, have_existing_dst_container) = self._ExpandDstUri(
+         self.args[-1])
+    name_expansion_iterator = NameExpansionIterator(
+        self.command_name, self.proj_id_handler, self.headers, self.debug,
+        self.bucket_storage_uri_class, uri_strs,
+        self.recursion_requested or self.perform_mv,
+        have_existing_dst_container)
+
+    # Use a lock to ensure accurate statistics in the face of
+    # multi-threading/multi-processing.
+    stats_lock = threading.Lock()
+
+    # Tracks if any copies failed.
+    self.copy_failure_count = 0
+
+    # Start the clock.
+    start_time = time.time()
+
+    # Tuple of attributes to share/manage across multiple processes in
+    # parallel (-m) mode.
+    shared_attrs = ('copy_failure_count', 'total_bytes_transferred')
+
+    # Perform copy requests in parallel (-m) mode, if requested, using
+    # configured number of parallel processes and threads. Otherwise,
+    # perform requests with sequential function calls in current process.
+    self.Apply(_CopyFunc, name_expansion_iterator, _CopyExceptionHandler,
+               shared_attrs)
+    if self.debug:
+      print 'total_bytes_transferred:' + str(self.total_bytes_transferred)
+
+    end_time = time.time()
+    self.total_elapsed_time = end_time - start_time
+
+    # Sometimes, particularly when running unit tests, the total elapsed time
+    # is really small. On Windows, the timer resolution is too small and
+    # causes total_elapsed_time to be zero.
+    try:
+      float(self.total_bytes_transferred) / float(self.total_elapsed_time)
+    except ZeroDivisionError:
+      self.total_elapsed_time = 0.01
+
+    self.total_bytes_per_second = (float(self.total_bytes_transferred) /
+                                   float(self.total_elapsed_time))
+
+    if self.debug == 3:
+      # Note that this only counts the actual GET and PUT bytes for the copy
+      # - not any transfers for doing wildcard expansion, the initial HEAD
+      # request boto performs when doing a bucket.get_key() operation, etc.
+      if self.total_bytes_transferred != 0:
+        self.THREADED_LOGGER.info(
+            'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)',
+                self.total_bytes_transferred, self.total_elapsed_time,
+                MakeHumanReadable(self.total_bytes_per_second))
+    if self.copy_failure_count:
+      plural_str = ''
+      if self.copy_failure_count > 1:
+        plural_str = 's'
+      raise CommandException('%d file%s/object%s could not be transferred.' % (
+                             self.copy_failure_count, plural_str, plural_str))
+
+    return 0
+
+  def _ParseArgs(self):
+    self.perform_mv = False
+    self.exclude_symlinks = False
+    self.quiet = False
+    self.no_clobber = False
+    self.continue_on_error = False
+    self.daisy_chain = False
+    self.read_args_from_stdin = False
+    self.print_ver = False
+    # self.recursion_requested initialized in command.py (so can be checked
+    # in parent class for all commands).
+    if self.sub_opts:
+      for o, unused_a in self.sub_opts:
+        if o == '-c':
+          self.continue_on_error = True
+        elif o == '-D':
+          self.daisy_chain = True
+        elif o == '-e':
+          self.exclude_symlinks = True
+        elif o == '-I':
+          self.read_args_from_stdin = True
+        elif o == '-M':
+          # Note that we signal to the cp command to perform a move (copy
+          # followed by remove) and use directory-move naming rules by passing
+          # the undocumented (for internal use) -M option when running the cp
+          # command from mv.py.
+          self.perform_mv = True
+        elif o == '-n':
+          self.no_clobber = True
+        elif o == '-q':
+          self.quiet = True
+        elif o == '-r' or o == '-R':
+          self.recursion_requested = True
+        elif o == '-v':
+          self.print_ver = True
+
+  def _HandleStreamingDownload(self):
+    # Destination is <STDOUT>. Manipulate sys.stdout so as to redirect all
+    # debug messages to <STDERR>.
+    stdout_fp = sys.stdout
+    sys.stdout = sys.stderr
+    did_some_work = False
+    for uri_str in self.args[0:len(self.args)-1]:
+      for uri in self.WildcardIterator(uri_str).IterUris():
+        did_some_work = True
+        key = uri.get_key(False, self.headers)
+        (elapsed_time, bytes_transferred) = self._PerformDownloadToStream(
+            key, uri, stdout_fp, self.headers)
+        self.total_elapsed_time += elapsed_time
+        self.total_bytes_transferred += bytes_transferred
+    if not did_some_work:
+      raise CommandException('No URIs matched')
+    if self.debug == 3:
+      if self.total_bytes_transferred != 0:
+        self.THREADED_LOGGER.info(
+            'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)',
+                self.total_bytes_transferred, self.total_elapsed_time,
+                 MakeHumanReadable(float(self.total_bytes_transferred) /
+                                   float(self.total_elapsed_time)))
+
+  def _StdinIterator(self):
+    """A generator function that returns lines from stdin."""
+    for line in sys.stdin:
+      # Strip CRLF.
+      yield line.rstrip()
+
+  def _SrcDstSame(self, src_uri, dst_uri):
+    """Checks if src_uri and dst_uri represent the same object or file.
+
+    We don't handle anything about hard or symbolic links.
+
+    Args:
+      src_uri: Source StorageUri.
+      dst_uri: Destination StorageUri.
+
+    Returns:
+      Bool indicator.
+    """
+    if src_uri.is_file_uri() and dst_uri.is_file_uri():
+      # Translate a/b/./c to a/b/c, so src=dst comparison below works.
+      new_src_path = os.path.normpath(src_uri.object_name)
+      new_dst_path = os.path.normpath(dst_uri.object_name)
+      return (src_uri.clone_replace_name(new_src_path).uri ==
+              dst_uri.clone_replace_name(new_dst_path).uri)
+    else:
+      return (src_uri.uri == dst_uri.uri and
+              src_uri.generation == dst_uri.generation and
+              src_uri.version_id == dst_uri.version_id)
+
+  def _ShouldTreatDstUriAsBucketSubDir(self, have_multiple_srcs, dst_uri,
+                                       have_existing_dest_subdir):
+    """
+    Checks whether dst_uri should be treated as a bucket "sub-directory". The
+    decision about whether something constitutes a bucket "sub-directory"
+    depends on whether there are multiple sources in this request and whether
+    there is an existing bucket subdirectory. For example, when running the
+    command:
+      gsutil cp file gs://bucket/abc
+    if there's no existing gs://bucket/abc bucket subdirectory we should copy
+    file to the object gs://bucket/abc. In contrast, if
+    there's an existing gs://bucket/abc bucket subdirectory we should copy
+    file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc
+    exists, when running the command:
+      gsutil cp file1 file2 gs://bucket/abc
+    we should copy file1 to gs://bucket/abc/file1 (and similarly for file2).
+
+    Note that we don't disallow naming a bucket "sub-directory" where there's
+    already an object at that URI. For example it's legitimate (albeit
+    confusing) to have an object called gs://bucket/dir and
+    then run the command
+    gsutil cp file1 file2 gs://bucket/dir
+    Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1,
+    and gs://bucket/dir/file2.
+
+    Args:
+      have_multiple_srcs: Bool indicator of whether this is a multi-source
+          operation.
+      dst_uri: StorageUri to check.
+      have_existing_dest_subdir: bool indicator whether dest is an existing
+        subdirectory.
+
+    Returns:
+      bool indicator.
+    """
+    return ((have_multiple_srcs and dst_uri.is_cloud_uri())
+            or (have_existing_dest_subdir))
+
+  def _ShouldTreatDstUriAsSingleton(self, have_multiple_srcs,
+                                    have_existing_dest_subdir, dst_uri):
+    """
+    Checks that dst_uri names a singleton (file or object) after
+    dir/wildcard expansion. The decision is more nuanced than simply
+    dst_uri.names_singleton()) because of the possibility that an object path
+    might name a bucket sub-directory.
+
+    Args:
+      have_multiple_srcs: Bool indicator of whether this is a multi-source
+          operation.
+      have_existing_dest_subdir: bool indicator whether dest is an existing
+        subdirectory.
+      dst_uri: StorageUri to check.
+
+    Returns:
+      bool indicator.
+    """
+    if have_multiple_srcs:
+      # Only a file meets the criteria in this case.
+      return dst_uri.names_file()
+    return not have_existing_dest_subdir and dst_uri.names_singleton()
+
+  def _IsNoClobberServerException(self, e):
+    """
+    Checks to see if the server attempted to clobber a file after we specified
+    in the header that we didn't want the file clobbered.
+
+    Args:
+      e: The Exception that was generated by a failed copy operation
+
+    Returns:
+      bool indicator - True indicates that the server did attempt to clobber
+          an existing file.
+    """
+    return self.no_clobber and (
+        (isinstance(e, GSResponseError) and e.status==412) or
+        (isinstance(e, ResumableUploadException) and 'code 412' in e.message))
+
+def _GetPathBeforeFinalDir(uri):
+  """
+  Returns the part of the path before the final directory component for the
+  given URI, handling cases for file system directories, bucket, and bucket
+  subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket',
+  and for file://dir we'll return file://
+
+  Args:
+    uri: StorageUri.
+
+  Returns:
+    String name of above-described path, sans final path separator.
+  """
+  sep = uri.delim
+  assert not uri.names_file()
+  if uri.names_directory():
+    past_scheme = uri.uri[len('file://'):]
+    if past_scheme.find(sep) == -1:
+      return 'file://'
+    else:
+      return 'file://%s' % past_scheme.rstrip(sep).rpartition(sep)[0]
+  if uri.names_bucket():
+    return '%s://' % uri.scheme
+  # Else it names a bucket subdir.
+  return uri.uri.rstrip(sep).rpartition(sep)[0]
+
+def _hash_filename(filename):
+  """
+  Apply a hash function (SHA1) to shorten the passed file name. The spec
+  for the hashed file name is as follows:
+
+      TRACKER_<hash>_<trailing>
+
+  where hash is a SHA1 hash on the original file name and trailing is
+  the last 16 chars from the original file name. Max file name lengths
+  vary by operating system so the goal of this function is to ensure
+  the hashed version takes fewer than 100 characters.
+
+  Args:
+    filename: file name to be hashed.
+
+  Returns:
+    shorter, hashed version of passed file name
+  """
+  if not isinstance(filename, unicode):
+    filename = unicode(filename, 'utf8').encode('utf-8')
+  m = hashlib.sha1(filename)
+  return "TRACKER_" + m.hexdigest() + '.' + filename[-16:]