Added gsutil to depot tools

third_party/gsutil/boto/boto/connection.py

Index: third_party/gsutil/boto/boto/connection.py
diff --git a/third_party/gsutil/boto/boto/connection.py b/third_party/gsutil/boto/boto/connection.py
new file mode 100644
index 0000000000000000000000000000000000000000..15ee4614878593851e3eb571e6e9f1f0c3ba3795
--- /dev/null
+++ b/third_party/gsutil/boto/boto/connection.py
@@ -0,0 +1,1081 @@
+# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.
+# Copyright (c) 2010 Google
+# Copyright (c) 2008 rPath, Inc.
+# Copyright (c) 2009 The Echo Nest Corporation
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+# Copyright (c) 2011, Nexenta Systems Inc.
+# All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+#
+# Parts of this code were copied or derived from sample code supplied by AWS.
+# The following notice applies to that code.
+#
+#  This software code is made available "AS IS" without warranties of any
+#  kind.  You may copy, display, modify and redistribute the software
+#  code either by itself or as incorporated into your code; provided that
+#  you do not remove any proprietary notices.  Your use of this software
+#  code is at your own risk and you waive any claim against Amazon
+#  Digital Services, Inc. or its affiliates with respect to your use of
+#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
+#  affiliates.
+
+"""
+Handles basic connections to AWS
+"""
+
+from __future__ import with_statement
+import base64
+import errno
+import httplib
+import os
+import Queue
+import random
+import re
+import socket
+import sys
+import time
+import urllib
+import urlparse
+import xml.sax
+import copy
+
+import auth
+import auth_handler
+import boto
+import boto.utils
+import boto.handler
+import boto.cacerts
+
+from boto import config, UserAgent
+from boto.exception import AWSConnectionError, BotoClientError
+from boto.exception import BotoServerError
+from boto.provider import Provider
+from boto.resultset import ResultSet
+
+HAVE_HTTPS_CONNECTION = False
+try:
+    import ssl
+    from boto import https_connection
+    # Google App Engine runs on Python 2.5 so doesn't have ssl.SSLError.
+    if hasattr(ssl, 'SSLError'):
+        HAVE_HTTPS_CONNECTION = True
+except ImportError:
+    pass
+
+try:
+    import threading
+except ImportError:
+    import dummy_threading as threading
+
+ON_APP_ENGINE = all(key in os.environ for key in (
+    'USER_IS_ADMIN', 'CURRENT_VERSION_ID', 'APPLICATION_ID'))
+
+PORTS_BY_SECURITY = {True: 443,
+                     False: 80}
+
+DEFAULT_CA_CERTS_FILE = os.path.join(os.path.dirname(os.path.abspath(boto.cacerts.__file__ )), "cacerts.txt")
+
+
+class HostConnectionPool(object):
+
+    """
+    A pool of connections for one remote (host,is_secure).
+
+    When connections are added to the pool, they are put into a
+    pending queue.  The _mexe method returns connections to the pool
+    before the response body has been read, so they connections aren't
+    ready to send another request yet.  They stay in the pending queue
+    until they are ready for another request, at which point they are
+    returned to the pool of ready connections.
+
+    The pool of ready connections is an ordered list of
+    (connection,time) pairs, where the time is the time the connection
+    was returned from _mexe.  After a certain period of time,
+    connections are considered stale, and discarded rather than being
+    reused.  This saves having to wait for the connection to time out
+    if AWS has decided to close it on the other end because of
+    inactivity.
+
+    Thread Safety:
+
+        This class is used only fram ConnectionPool while it's mutex
+        is held.
+    """
+
+    def __init__(self):
+        self.queue = []
+
+    def size(self):
+        """
+        Returns the number of connections in the pool for this host.
+        Some of the connections may still be in use, and may not be
+        ready to be returned by get().
+        """
+        return len(self.queue)
+
+    def put(self, conn):
+        """
+        Adds a connection to the pool, along with the time it was
+        added.
+        """
+        self.queue.append((conn, time.time()))
+
+    def get(self):
+        """
+        Returns the next connection in this pool that is ready to be
+        reused.  Returns None of there aren't any.
+        """
+        # Discard ready connections that are too old.
+        self.clean()
+
+        # Return the first connection that is ready, and remove it
+        # from the queue.  Connections that aren't ready are returned
+        # to the end of the queue with an updated time, on the
+        # assumption that somebody is actively reading the response.
+        for _ in range(len(self.queue)):
+            (conn, _) = self.queue.pop(0)
+            if self._conn_ready(conn):
+                return conn
+            else:
+                self.put(conn)
+        return None
+
+    def _conn_ready(self, conn):
+        """
+        There is a nice state diagram at the top of httplib.py.  It
+        indicates that once the response headers have been read (which
+        _mexe does before adding the connection to the pool), a
+        response is attached to the connection, and it stays there
+        until it's done reading.  This isn't entirely true: even after
+        the client is done reading, the response may be closed, but
+        not removed from the connection yet.
+
+        This is ugly, reading a private instance variable, but the
+        state we care about isn't available in any public methods.
+        """
+        if ON_APP_ENGINE:
+            # Google AppEngine implementation of HTTPConnection doesn't contain
+            # _HTTPConnection__response attribute. Moreover, it's not possible
+            # to determine if given connection is ready. Reusing connections
+            # simply doesn't make sense with App Engine urlfetch service.
+            return False
+        else:
+            response = getattr(conn, '_HTTPConnection__response', None)
+            return (response is None) or response.isclosed()
+
+    def clean(self):
+        """
+        Get rid of stale connections.
+        """
+        # Note that we do not close the connection here -- somebody
+        # may still be reading from it.
+        while len(self.queue) > 0 and self._pair_stale(self.queue[0]):
+            self.queue.pop(0)
+
+    def _pair_stale(self, pair):
+        """
+        Returns true of the (connection,time) pair is too old to be
+        used.
+        """
+        (_conn, return_time) = pair
+        now = time.time()
+        return return_time + ConnectionPool.STALE_DURATION < now
+
+
+class ConnectionPool(object):
+
+    """
+    A connection pool that expires connections after a fixed period of
+    time.  This saves time spent waiting for a connection that AWS has
+    timed out on the other end.
+
+    This class is thread-safe.
+    """
+
+    #
+    # The amout of time between calls to clean.
+    #
+
+    CLEAN_INTERVAL = 5.0
+
+    #
+    # How long before a connection becomes "stale" and won't be reused
+    # again.  The intention is that this time is less that the timeout
+    # period that AWS uses, so we'll never try to reuse a connection
+    # and find that AWS is timing it out.
+    #
+    # Experimentation in July 2011 shows that AWS starts timing things
+    # out after three minutes.  The 60 seconds here is conservative so
+    # we should never hit that 3-minute timout.
+    #
+
+    STALE_DURATION = 60.0
+
+    def __init__(self):
+        # Mapping from (host,is_secure) to HostConnectionPool.
+        # If a pool becomes empty, it is removed.
+        self.host_to_pool = {}
+        # The last time the pool was cleaned.
+        self.last_clean_time = 0.0
+        self.mutex = threading.Lock()
+        ConnectionPool.STALE_DURATION = \
+            config.getfloat('Boto', 'connection_stale_duration',
+                            ConnectionPool.STALE_DURATION)
+
+    def __getstate__(self):
+        pickled_dict = copy.copy(self.__dict__)
+        pickled_dict['host_to_pool'] = {}
+        del pickled_dict['mutex']
+        return pickled_dict
+
+    def __setstate__(self, dct):
+        self.__init__()
+
+    def size(self):
+        """
+        Returns the number of connections in the pool.
+        """
+        return sum(pool.size() for pool in self.host_to_pool.values())
+
+    def get_http_connection(self, host, is_secure):
+        """
+        Gets a connection from the pool for the named host.  Returns
+        None if there is no connection that can be reused. It's the caller's
+        responsibility to call close() on the connection when it's no longer
+        needed.
+        """
+        self.clean()
+        with self.mutex:
+            key = (host, is_secure)
+            if key not in self.host_to_pool:
+                return None
+            return self.host_to_pool[key].get()
+
+    def put_http_connection(self, host, is_secure, conn):
+        """
+        Adds a connection to the pool of connections that can be
+        reused for the named host.
+        """
+        with self.mutex:
+            key = (host, is_secure)
+            if key not in self.host_to_pool:
+                self.host_to_pool[key] = HostConnectionPool()
+            self.host_to_pool[key].put(conn)
+
+    def clean(self):
+        """
+        Clean up the stale connections in all of the pools, and then
+        get rid of empty pools.  Pools clean themselves every time a
+        connection is fetched; this cleaning takes care of pools that
+        aren't being used any more, so nothing is being gotten from
+        them.
+        """
+        with self.mutex:
+            now = time.time()
+            if self.last_clean_time + self.CLEAN_INTERVAL < now:
+                to_remove = []
+                for (host, pool) in self.host_to_pool.items():
+                    pool.clean()
+                    if pool.size() == 0:
+                        to_remove.append(host)
+                for host in to_remove:
+                    del self.host_to_pool[host]
+                self.last_clean_time = now
+
+
+class HTTPRequest(object):
+
+    def __init__(self, method, protocol, host, port, path, auth_path,
+                 params, headers, body):
+        """Represents an HTTP request.
+
+        :type method: string
+        :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc.
+
+        :type protocol: string
+        :param protocol: The http protocol used, 'http' or 'https'.
+
+        :type host: string
+        :param host: Host to which the request is addressed. eg. abc.com
+
+        :type port: int
+        :param port: port on which the request is being sent. Zero means unset,
+            in which case default port will be chosen.
+
+        :type path: string
+        :param path: URL path that is being accessed.
+
+        :type auth_path: string
+        :param path: The part of the URL path used when creating the
+            authentication string.
+
+        :type params: dict
+        :param params: HTTP url query parameters, with key as name of
+            the param, and value as value of param.
+
+        :type headers: dict
+        :param headers: HTTP headers, with key as name of the header and value
+            as value of header.
+
+        :type body: string
+        :param body: Body of the HTTP request. If not present, will be None or
+            empty string ('').
+        """
+        self.method = method
+        self.protocol = protocol
+        self.host = host
+        self.port = port
+        self.path = path
+        if auth_path is None:
+            auth_path = path
+        self.auth_path = auth_path
+        self.params = params
+        # chunked Transfer-Encoding should act only on PUT request.
+        if headers and 'Transfer-Encoding' in headers and \
+                headers['Transfer-Encoding'] == 'chunked' and \
+                self.method != 'PUT':
+            self.headers = headers.copy()
+            del self.headers['Transfer-Encoding']
+        else:
+            self.headers = headers
+        self.body = body
+
+    def __str__(self):
+        return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) '
+                 'params(%s) headers(%s) body(%s)') % (self.method,
+                 self.protocol, self.host, self.port, self.path, self.params,
+                 self.headers, self.body))
+
+    def authorize(self, connection, **kwargs):
+        for key in self.headers:
+            val = self.headers[key]
+            if isinstance(val, unicode):
+                self.headers[key] = urllib.quote_plus(val.encode('utf-8'))
+
+        connection._auth_handler.add_auth(self, **kwargs)
+
+        self.headers['User-Agent'] = UserAgent
+        # I'm not sure if this is still needed, now that add_auth is
+        # setting the content-length for POST requests.
+        if 'Content-Length' not in self.headers:
+            if 'Transfer-Encoding' not in self.headers or \
+                    self.headers['Transfer-Encoding'] != 'chunked':
+                self.headers['Content-Length'] = str(len(self.body))
+
+
+class HTTPResponse(httplib.HTTPResponse):
+
+    def __init__(self, *args, **kwargs):
+        httplib.HTTPResponse.__init__(self, *args, **kwargs)
+        self._cached_response = ''
+
+    def read(self, amt=None):
+        """Read the response.
+
+        This method does not have the same behavior as
+        httplib.HTTPResponse.read.  Instead, if this method is called with
+        no ``amt`` arg, then the response body will be cached.  Subsequent
+        calls to ``read()`` with no args **will return the cached response**.
+
+        """
+        if amt is None:
+            # The reason for doing this is that many places in boto call
+            # response.read() and except to get the response body that they
+            # can then process.  To make sure this always works as they expect
+            # we're caching the response so that multiple calls to read()
+            # will return the full body.  Note that this behavior only
+            # happens if the amt arg is not specified.
+            if not self._cached_response:
+                self._cached_response = httplib.HTTPResponse.read(self)
+            return self._cached_response
+        else:
+            return httplib.HTTPResponse.read(self, amt)
+
+
+class AWSAuthConnection(object):
+    def __init__(self, host, aws_access_key_id=None,
+                 aws_secret_access_key=None,
+                 is_secure=True, port=None, proxy=None, proxy_port=None,
+                 proxy_user=None, proxy_pass=None, debug=0,
+                 https_connection_factory=None, path='/',
+                 provider='aws', security_token=None,
+                 suppress_consec_slashes=True,
+                 validate_certs=True):
+        """
+        :type host: str
+        :param host: The host to make the connection to
+
+        :keyword str aws_access_key_id: Your AWS Access Key ID (provided by
+            Amazon). If none is specified, the value in your
+            ``AWS_ACCESS_KEY_ID`` environmental variable is used.
+        :keyword str aws_secret_access_key: Your AWS Secret Access Key
+            (provided by Amazon). If none is specified, the value in your
+            ``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
+
+        :type is_secure: boolean
+        :param is_secure: Whether the connection is over SSL
+
+        :type https_connection_factory: list or tuple
+        :param https_connection_factory: A pair of an HTTP connection
+            factory and the exceptions to catch.  The factory should have
+            a similar interface to L{httplib.HTTPSConnection}.
+
+        :param str proxy: Address/hostname for a proxy server
+
+        :type proxy_port: int
+        :param proxy_port: The port to use when connecting over a proxy
+
+        :type proxy_user: str
+        :param proxy_user: The username to connect with on the proxy
+
+        :type proxy_pass: str
+        :param proxy_pass: The password to use when connection over a proxy.
+
+        :type port: int
+        :param port: The port to use to connect
+
+        :type suppress_consec_slashes: bool
+        :param suppress_consec_slashes: If provided, controls whether
+            consecutive slashes will be suppressed in key paths.
+
+        :type validate_certs: bool
+        :param validate_certs: Controls whether SSL certificates
+            will be validated or not.  Defaults to True.
+        """
+        self.suppress_consec_slashes = suppress_consec_slashes
+        self.num_retries = 6
+        # Override passed-in is_secure setting if value was defined in config.
+        if config.has_option('Boto', 'is_secure'):
+            is_secure = config.getboolean('Boto', 'is_secure')
+        self.is_secure = is_secure
+        # Whether or not to validate server certificates.
+        # The default is now to validate certificates.  This can be
+        # overridden in the boto config file are by passing an
+        # explicit validate_certs parameter to the class constructor.
+        self.https_validate_certificates = config.getbool(
+            'Boto', 'https_validate_certificates',
+            validate_certs)
+        if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
+            raise BotoClientError(
+                    "SSL server certificate validation is enabled in boto "
+                    "configuration, but Python dependencies required to "
+                    "support this feature are not available. Certificate "
+                    "validation is only supported when running under Python "
+                    "2.6 or later.")
+        self.ca_certificates_file = config.get_value(
+                'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
+        self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
+        # define exceptions from httplib that we want to catch and retry
+        self.http_exceptions = (httplib.HTTPException, socket.error,
+                                socket.gaierror, httplib.BadStatusLine)
+        # define subclasses of the above that are not retryable.
+        self.http_unretryable_exceptions = []
+        if HAVE_HTTPS_CONNECTION:
+            self.http_unretryable_exceptions.append(
+                    https_connection.InvalidCertificateException)
+
+        # define values in socket exceptions we don't want to catch
+        self.socket_exception_values = (errno.EINTR,)
+        if https_connection_factory is not None:
+            self.https_connection_factory = https_connection_factory[0]
+            self.http_exceptions += https_connection_factory[1]
+        else:
+            self.https_connection_factory = None
+        if (is_secure):
+            self.protocol = 'https'
+        else:
+            self.protocol = 'http'
+        self.host = host
+        self.path = path
+        # if the value passed in for debug
+        if not isinstance(debug, (int, long)):
+            debug = 0
+        self.debug = config.getint('Boto', 'debug', debug)
+        if port:
+            self.port = port
+        else:
+            self.port = PORTS_BY_SECURITY[is_secure]
+
+        # Timeout used to tell httplib how long to wait for socket timeouts.
+        # Default is to leave timeout unchanged, which will in turn result in
+        # the socket's default global timeout being used. To specify a
+        # timeout, set http_socket_timeout in Boto config. Regardless,
+        # timeouts will only be applied if Python is 2.6 or greater.
+        self.http_connection_kwargs = {}
+        if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
+            if config.has_option('Boto', 'http_socket_timeout'):
+                timeout = config.getint('Boto', 'http_socket_timeout')
+                self.http_connection_kwargs['timeout'] = timeout
+
+        if isinstance(provider, Provider):
+            # Allow overriding Provider
+            self.provider = provider
+        else:
+            self._provider_type = provider
+            self.provider = Provider(self._provider_type,
+                                     aws_access_key_id,
+                                     aws_secret_access_key,
+                                     security_token)
+
+        # allow config file to override default host
+        if self.provider.host:
+            self.host = self.provider.host
+
+        self._pool = ConnectionPool()
+        self._connection = (self.server_name(), self.is_secure)
+        self._last_rs = None
+        self._auth_handler = auth.get_auth_handler(
+              host, config, self.provider, self._required_auth_capability())
+        if getattr(self, 'AuthServiceName', None) is not None:
+            self.auth_service_name = self.AuthServiceName
+
+    def __repr__(self):
+        return '%s:%s' % (self.__class__.__name__, self.host)
+
+    def _required_auth_capability(self):
+        return []
+
+    def _get_auth_service_name(self):
+        return getattr(self._auth_handler, 'service_name')
+
+    # For Sigv4, the auth_service_name/auth_region_name properties allow
+    # the service_name/region_name to be explicitly set instead of being
+    # derived from the endpoint url.
+    def _set_auth_service_name(self, value):
+        self._auth_handler.service_name = value
+    auth_service_name = property(_get_auth_service_name, _set_auth_service_name)
+
+    def _get_auth_region_name(self):
+        return getattr(self._auth_handler, 'region_name')
+
+    def _set_auth_region_name(self, value):
+        self._auth_handler.region_name = value
+    auth_region_name = property(_get_auth_region_name, _set_auth_region_name)
+
+    def connection(self):
+        return self.get_http_connection(*self._connection)
+    connection = property(connection)
+
+    def aws_access_key_id(self):
+        return self.provider.access_key
+    aws_access_key_id = property(aws_access_key_id)
+    gs_access_key_id = aws_access_key_id
+    access_key = aws_access_key_id
+
+    def aws_secret_access_key(self):
+        return self.provider.secret_key
+    aws_secret_access_key = property(aws_secret_access_key)
+    gs_secret_access_key = aws_secret_access_key
+    secret_key = aws_secret_access_key
+
+    def get_path(self, path='/'):
+        # The default behavior is to suppress consecutive slashes for reasons
+        # discussed at
+        # https://groups.google.com/forum/#!topic/boto-dev/-ft0XPUy0y8
+        # You can override that behavior with the suppress_consec_slashes param.
+        if not self.suppress_consec_slashes:
+            return self.path + re.sub('^/*', "", path)
+        pos = path.find('?')
+        if pos >= 0:
+            params = path[pos:]
+            path = path[:pos]
+        else:
+            params = None
+        if path[-1] == '/':
+            need_trailing = True
+        else:
+            need_trailing = False
+        path_elements = self.path.split('/')
+        path_elements.extend(path.split('/'))
+        path_elements = [p for p in path_elements if p]
+        path = '/' + '/'.join(path_elements)
+        if path[-1] != '/' and need_trailing:
+            path += '/'
+        if params:
+            path = path + params
+        return path
+
+    def server_name(self, port=None):
+        if not port:
+            port = self.port
+        if port == 80:
+            signature_host = self.host
+        else:
+            # This unfortunate little hack can be attributed to
+            # a difference in the 2.6 version of httplib.  In old
+            # versions, it would append ":443" to the hostname sent
+            # in the Host header and so we needed to make sure we
+            # did the same when calculating the V2 signature.  In 2.6
+            # (and higher!)
+            # it no longer does that.  Hence, this kludge.
+            if ((ON_APP_ENGINE and sys.version[:3] == '2.5') or
+                    sys.version[:3] in ('2.6', '2.7')) and port == 443:
+                signature_host = self.host
+            else:
+                signature_host = '%s:%d' % (self.host, port)
+        return signature_host
+
+    def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
+        self.proxy = proxy
+        self.proxy_port = proxy_port
+        self.proxy_user = proxy_user
+        self.proxy_pass = proxy_pass
+        if 'http_proxy' in os.environ and not self.proxy:
+            pattern = re.compile(
+                '(?:http://)?' \
+                '(?:(?P<user>\w+):(?P<pass>.*)@)?' \
+                '(?P<host>[\w\-\.]+)' \
+                '(?::(?P<port>\d+))?'
+            )
+            match = pattern.match(os.environ['http_proxy'])
+            if match:
+                self.proxy = match.group('host')
+                self.proxy_port = match.group('port')
+                self.proxy_user = match.group('user')
+                self.proxy_pass = match.group('pass')
+        else:
+            if not self.proxy:
+                self.proxy = config.get_value('Boto', 'proxy', None)
+            if not self.proxy_port:
+                self.proxy_port = config.get_value('Boto', 'proxy_port', None)
+            if not self.proxy_user:
+                self.proxy_user = config.get_value('Boto', 'proxy_user', None)
+            if not self.proxy_pass:
+                self.proxy_pass = config.get_value('Boto', 'proxy_pass', None)
+
+        if not self.proxy_port and self.proxy:
+            print "http_proxy environment variable does not specify " \
+                "a port, using default"
+            self.proxy_port = self.port
+        self.use_proxy = (self.proxy != None)
+
+    def get_http_connection(self, host, is_secure):
+        conn = self._pool.get_http_connection(host, is_secure)
+        if conn is not None:
+            return conn
+        else:
+            return self.new_http_connection(host, is_secure)
+
+    def new_http_connection(self, host, is_secure):
+        if self.use_proxy and not is_secure:
+            host = '%s:%d' % (self.proxy, int(self.proxy_port))
+        if host is None:
+            host = self.server_name()
+        if is_secure:
+            boto.log.debug(
+                    'establishing HTTPS connection: host=%s, kwargs=%s',
+                    host, self.http_connection_kwargs)
+            if self.use_proxy:
+                connection = self.proxy_ssl(host, is_secure and 443 or 80)
+            elif self.https_connection_factory:
+                connection = self.https_connection_factory(host)
+            elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
+                connection = https_connection.CertValidatingHTTPSConnection(
+                        host, ca_certs=self.ca_certificates_file,
+                        **self.http_connection_kwargs)
+            else:
+                connection = httplib.HTTPSConnection(host,
+                        **self.http_connection_kwargs)
+        else:
+            boto.log.debug('establishing HTTP connection: kwargs=%s' %
+                    self.http_connection_kwargs)
+            if self.https_connection_factory:
+                # even though the factory says https, this is too handy
+                # to not be able to allow overriding for http also.
+                connection = self.https_connection_factory(host,
+                    **self.http_connection_kwargs)
+            else:
+                connection = httplib.HTTPConnection(host,
+                    **self.http_connection_kwargs)
+        if self.debug > 1:
+            connection.set_debuglevel(self.debug)
+        # self.connection must be maintained for backwards-compatibility
+        # however, it must be dynamically pulled from the connection pool
+        # set a private variable which will enable that
+        if host.split(':')[0] == self.host and is_secure == self.is_secure:
+            self._connection = (host, is_secure)
+        # Set the response class of the http connection to use our custom
+        # class.
+        connection.response_class = HTTPResponse
+        return connection
+
+    def put_http_connection(self, host, is_secure, connection):
+        self._pool.put_http_connection(host, is_secure, connection)
+
+    def proxy_ssl(self, host=None, port=None):
+        if host and port:
+            host = '%s:%d' % (host, port)
+        else:
+            host = '%s:%d' % (self.host, self.port)
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            sock.connect((self.proxy, int(self.proxy_port)))
+        except:
+            raise
+        boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host)
+        sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
+        sock.sendall("User-Agent: %s\r\n" % UserAgent)
+        if self.proxy_user and self.proxy_pass:
+            for k, v in self.get_proxy_auth_header().items():
+                sock.sendall("%s: %s\r\n" % (k, v))
+            # See discussion about this config option at
+            # https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc
+            if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False):
+                sock.sendall("\r\n")
+        else:
+            sock.sendall("\r\n")
+        resp = httplib.HTTPResponse(sock, strict=True, debuglevel=self.debug)
+        resp.begin()
+
+        if resp.status != 200:
+            # Fake a socket error, use a code that make it obvious it hasn't
+            # been generated by the socket library
+            raise socket.error(-71,
+                               "Error talking to HTTP proxy %s:%s: %s (%s)" %
+                               (self.proxy, self.proxy_port,
+                                resp.status, resp.reason))
+
+        # We can safely close the response, it duped the original socket
+        resp.close()
+
+        h = httplib.HTTPConnection(host)
+
+        if self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
+            boto.log.debug("wrapping ssl socket for proxied connection; "
+                           "CA certificate file=%s",
+                           self.ca_certificates_file)
+            key_file = self.http_connection_kwargs.get('key_file', None)
+            cert_file = self.http_connection_kwargs.get('cert_file', None)
+            sslSock = ssl.wrap_socket(sock, keyfile=key_file,
+                                      certfile=cert_file,
+                                      cert_reqs=ssl.CERT_REQUIRED,
+                                      ca_certs=self.ca_certificates_file)
+            cert = sslSock.getpeercert()
+            hostname = self.host.split(':', 0)[0]
+            if not https_connection.ValidateCertificateHostname(cert, hostname):
+                raise https_connection.InvalidCertificateException(
+                        hostname, cert, 'hostname mismatch')
+        else:
+            # Fallback for old Python without ssl.wrap_socket
+            if hasattr(httplib, 'ssl'):
+                sslSock = httplib.ssl.SSLSocket(sock)
+            else:
+                sslSock = socket.ssl(sock, None, None)
+                sslSock = httplib.FakeSocket(sock, sslSock)
+
+        # This is a bit unclean
+        h.sock = sslSock
+        return h
+
+    def prefix_proxy_to_path(self, path, host=None):
+        path = self.protocol + '://' + (host or self.server_name()) + path
+        return path
+
+    def get_proxy_auth_header(self):
+        auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass)
+        return {'Proxy-Authorization': 'Basic %s' % auth}
+
+    def _mexe(self, request, sender=None, override_num_retries=None,
+              retry_handler=None):
+        """
+        mexe - Multi-execute inside a loop, retrying multiple times to handle
+               transient Internet errors by simply trying again.
+               Also handles redirects.
+
+        This code was inspired by the S3Utils classes posted to the boto-users
+        Google group by Larry Bates.  Thanks!
+
+        """
+        boto.log.debug('Method: %s' % request.method)
+        boto.log.debug('Path: %s' % request.path)
+        boto.log.debug('Data: %s' % request.body)
+        boto.log.debug('Headers: %s' % request.headers)
+        boto.log.debug('Host: %s' % request.host)
+        response = None
+        body = None
+        e = None
+        if override_num_retries is None:
+            num_retries = config.getint('Boto', 'num_retries', self.num_retries)
+        else:
+            num_retries = override_num_retries
+        i = 0
+        connection = self.get_http_connection(request.host, self.is_secure)
+        while i <= num_retries:
+            # Use binary exponential backoff to desynchronize client requests.
+            next_sleep = random.random() * (2 ** i)
+            try:
+                # we now re-sign each request before it is retried
+                boto.log.debug('Token: %s' % self.provider.security_token)
+                request.authorize(connection=self)
+                if callable(sender):
+                    response = sender(connection, request.method, request.path,
+                                      request.body, request.headers)
+                else:
+                    connection.request(request.method, request.path,
+                                       request.body, request.headers)
+                    response = connection.getresponse()
+                location = response.getheader('location')
+                # -- gross hack --
+                # httplib gets confused with chunked responses to HEAD requests
+                # so I have to fake it out
+                if request.method == 'HEAD' and getattr(response,
+                                                        'chunked', False):
+                    response.chunked = 0
+                if callable(retry_handler):
+                    status = retry_handler(response, i, next_sleep)
+                    if status:
+                        msg, i, next_sleep = status
+                        if msg:
+                            boto.log.debug(msg)
+                        time.sleep(next_sleep)
+                        continue
+                if response.status == 500 or response.status == 503:
+                    msg = 'Received %d response.  ' % response.status
+                    msg += 'Retrying in %3.1f seconds' % next_sleep
+                    boto.log.debug(msg)
+                    body = response.read()
+                elif response.status < 300 or response.status >= 400 or \
+                        not location:
+                    self.put_http_connection(request.host, self.is_secure,
+                                             connection)
+                    return response
+                else:
+                    scheme, request.host, request.path, \
+                        params, query, fragment = urlparse.urlparse(location)
+                    if query:
+                        request.path += '?' + query
+                    msg = 'Redirecting: %s' % scheme + '://'
+                    msg += request.host + request.path
+                    boto.log.debug(msg)
+                    connection = self.get_http_connection(request.host,
+                                                          scheme == 'https')
+                    response = None
+                    continue
+            except self.http_exceptions, e:
+                for unretryable in self.http_unretryable_exceptions:
+                    if isinstance(e, unretryable):
+                        boto.log.debug(
+                            'encountered unretryable %s exception, re-raising' %
+                            e.__class__.__name__)
+                        raise e
+                boto.log.debug('encountered %s exception, reconnecting' % \
+                                  e.__class__.__name__)
+                connection = self.new_http_connection(request.host,
+                                                      self.is_secure)
+            time.sleep(next_sleep)
+            i += 1
+        # If we made it here, it's because we have exhausted our retries
+        # and stil haven't succeeded.  So, if we have a response object,
+        # use it to raise an exception.
+        # Otherwise, raise the exception that must have already h#appened.
+        if response:
+            raise BotoServerError(response.status, response.reason, body)
+        elif e:
+            raise e
+        else:
+            msg = 'Please report this exception as a Boto Issue!'
+            raise BotoClientError(msg)
+
+    def build_base_http_request(self, method, path, auth_path,
+                                params=None, headers=None, data='', host=None):
+        path = self.get_path(path)
+        if auth_path is not None:
+            auth_path = self.get_path(auth_path)
+        if params == None:
+            params = {}
+        else:
+            params = params.copy()
+        if headers == None:
+            headers = {}
+        else:
+            headers = headers.copy()
+        host = host or self.host
+        if self.use_proxy:
+            if not auth_path:
+                auth_path = path
+            path = self.prefix_proxy_to_path(path, host)
+            if self.proxy_user and self.proxy_pass and not self.is_secure:
+                # If is_secure, we don't have to set the proxy authentication
+                # header here, we did that in the CONNECT to the proxy.
+                headers.update(self.get_proxy_auth_header())
+        return HTTPRequest(method, self.protocol, host, self.port,
+                           path, auth_path, params, headers, data)
+
+    def make_request(self, method, path, headers=None, data='', host=None,
+                     auth_path=None, sender=None, override_num_retries=None,
+                     params=None):
+        """Makes a request to the server, with stock multiple-retry logic."""
+        if params is None:
+            params = {}
+        http_request = self.build_base_http_request(method, path, auth_path,
+                                                    params, headers, data, host)
+        return self._mexe(http_request, sender, override_num_retries)
+
+    def close(self):
+        """(Optional) Close any open HTTP connections.  This is non-destructive,
+        and making a new request will open a connection again."""
+
+        boto.log.debug('closing all HTTP connections')
+        self._connection = None  # compat field
+
+
+class AWSQueryConnection(AWSAuthConnection):
+
+    APIVersion = ''
+    ResponseError = BotoServerError
+
+    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
+                 is_secure=True, port=None, proxy=None, proxy_port=None,
+                 proxy_user=None, proxy_pass=None, host=None, debug=0,
+                 https_connection_factory=None, path='/', security_token=None,
+                 validate_certs=True):
+        AWSAuthConnection.__init__(self, host, aws_access_key_id,
+                                   aws_secret_access_key,
+                                   is_secure, port, proxy,
+                                   proxy_port, proxy_user, proxy_pass,
+                                   debug, https_connection_factory, path,
+                                   security_token=security_token,
+                                   validate_certs=validate_certs)
+
+    def _required_auth_capability(self):
+        return []
+
+    def get_utf8_value(self, value):
+        return boto.utils.get_utf8_value(value)
+
+    def make_request(self, action, params=None, path='/', verb='GET'):
+        http_request = self.build_base_http_request(verb, path, None,
+                                                    params, {}, '',
+                                                    self.server_name())
+        if action:
+            http_request.params['Action'] = action
+        if self.APIVersion:
+            http_request.params['Version'] = self.APIVersion
+        return self._mexe(http_request)
+
+    def build_list_params(self, params, items, label):
+        if isinstance(items, basestring):
+            items = [items]
+        for i in range(1, len(items) + 1):
+            params['%s.%d' % (label, i)] = items[i - 1]
+
+    def build_complex_list_params(self, params, items, label, names):
+        """Serialize a list of structures.
+
+        For example::
+
+            items = [('foo', 'bar', 'baz'), ('foo2', 'bar2', 'baz2')]
+            label = 'ParamName.member'
+            names = ('One', 'Two', 'Three')
+            self.build_complex_list_params(params, items, label, names)
+
+        would result in the params dict being updated with these params::
+
+            ParamName.member.1.One = foo
+            ParamName.member.1.Two = bar
+            ParamName.member.1.Three = baz
+
+            ParamName.member.2.One = foo2
+            ParamName.member.2.Two = bar2
+            ParamName.member.2.Three = baz2
+
+        :type params: dict
+        :param params: The params dict.  The complex list params
+            will be added to this dict.
+
+        :type items: list of tuples
+        :param items: The list to serialize.
+
+        :type label: string
+        :param label: The prefix to apply to the parameter.
+
+        :type names: tuple of strings
+        :param names: The names associated with each tuple element.
+
+        """
+        for i, item in enumerate(items, 1):
+            current_prefix = '%s.%s' % (label, i)
+            for key, value in zip(names, item):
+                full_key = '%s.%s' % (current_prefix, key)
+                params[full_key] = value
+
+    # generics
+
+    def get_list(self, action, params, markers, path='/',
+                 parent=None, verb='GET'):
+        if not parent:
+            parent = self
+        response = self.make_request(action, params, path, verb)
+        body = response.read()
+        boto.log.debug(body)
+        if not body:
+            boto.log.error('Null body %s' % body)
+            raise self.ResponseError(response.status, response.reason, body)
+        elif response.status == 200:
+            rs = ResultSet(markers)
+            h = boto.handler.XmlHandler(rs, parent)
+            xml.sax.parseString(body, h)
+            return rs
+        else:
+            boto.log.error('%s %s' % (response.status, response.reason))
+            boto.log.error('%s' % body)
+            raise self.ResponseError(response.status, response.reason, body)
+
+    def get_object(self, action, params, cls, path='/',
+                   parent=None, verb='GET'):
+        if not parent:
+            parent = self
+        response = self.make_request(action, params, path, verb)
+        body = response.read()
+        boto.log.debug(body)
+        if not body:
+            boto.log.error('Null body %s' % body)
+            raise self.ResponseError(response.status, response.reason, body)
+        elif response.status == 200:
+            obj = cls(parent)
+            h = boto.handler.XmlHandler(obj, parent)
+            xml.sax.parseString(body, h)
+            return obj
+        else:
+            boto.log.error('%s %s' % (response.status, response.reason))
+            boto.log.error('%s' % body)
+            raise self.ResponseError(response.status, response.reason, body)
+
+    def get_status(self, action, params, path='/', parent=None, verb='GET'):
+        if not parent:
+            parent = self
+        response = self.make_request(action, params, path, verb)
+        body = response.read()
+        boto.log.debug(body)
+        if not body:
+            boto.log.error('Null body %s' % body)
+            raise self.ResponseError(response.status, response.reason, body)
+        elif response.status == 200:
+            rs = ResultSet()
+            h = boto.handler.XmlHandler(rs, parent)
+            xml.sax.parseString(body, h)
+            return rs.status
+        else:
+            boto.log.error('%s %s' % (response.status, response.reason))
+            boto.log.error('%s' % body)
+            raise self.ResponseError(response.status, response.reason, body)