Upgrade gsutil to 3.4

third_party/gsutil/boto/boto/auth.py

Index: third_party/gsutil/boto/boto/auth.py
diff --git a/third_party/gsutil/20110627/boto/boto/auth.py b/third_party/gsutil/boto/boto/auth.py
similarity index 68%
rename from third_party/gsutil/20110627/boto/boto/auth.py
rename to third_party/gsutil/boto/boto/auth.py
index f27f9b4a10eb9898a9d5d80464bca79a6b19d454..14bd0fa261567d70298e5db579296e80b3b71754 100644
--- a/third_party/gsutil/20110627/boto/boto/auth.py
+++ b/third_party/gsutil/boto/boto/auth.py
@@ -34,8 +34,8 @@ import boto.plugin
 import boto.utils
 import hmac
 import sys
-import time
 import urllib
+from email.utils import formatdate
 
 from boto.auth_handler import AuthHandler
 from boto.exception import BotoClientError
@@ -74,10 +74,15 @@ class HmacKeys(object):
     def __init__(self, host, config, provider):
         if provider.access_key is None or provider.secret_key is None:
             raise boto.auth_handler.NotReadyToAuthenticate()
+        self.host = host
+        self.update_provider(provider)
+
+    def update_provider(self, provider):
         self._provider = provider
         self._hmac = hmac.new(self._provider.secret_key, digestmod=sha)
         if sha256:
-            self._hmac_256 = hmac.new(self._provider.secret_key, digestmod=sha256)
+            self._hmac_256 = hmac.new(self._provider.secret_key,
+                                      digestmod=sha256)
         else:
             self._hmac_256 = None
 
@@ -88,7 +93,6 @@ class HmacKeys(object):
             return 'HmacSHA1'
 
     def sign_string(self, string_to_sign):
-        boto.log.debug('Canonical: %s' % string_to_sign)
         if self._hmac_256:
             hmac = self._hmac_256.copy()
         else:
@@ -96,6 +100,19 @@ class HmacKeys(object):
         hmac.update(string_to_sign)
         return base64.encodestring(hmac.digest()).strip()
 
+class AnonAuthHandler(AuthHandler, HmacKeys):
+    """
+    Implements Anonymous requests.
+    """
+    
+    capability = ['anon']
+    
+    def __init__(self, host, config, provider):
+        AuthHandler.__init__(self, host, config, provider)
+        
+    def add_auth(self, http_request, **kwargs):
+        pass
+
 class HmacAuthV1Handler(AuthHandler, HmacKeys):
     """    Implements the HMAC request signing used by S3 and GS."""
     
@@ -111,12 +128,16 @@ class HmacAuthV1Handler(AuthHandler, HmacKeys):
         method = http_request.method
         auth_path = http_request.auth_path
         if not headers.has_key('Date'):
-            headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
-                                            time.gmtime())
-
-        c_string = boto.utils.canonical_string(method, auth_path, headers,
-                                               None, self._provider)
-        b64_hmac = self.sign_string(c_string)
+            headers['Date'] = formatdate(usegmt=True)
+
+        if self._provider.security_token:
+            key = self._provider.security_token_header
+            headers[key] = self._provider.security_token
+        string_to_sign = boto.utils.canonical_string(method, auth_path,
+                                                     headers, None,
+                                                     self._provider)
+        boto.log.debug('StringToSign:\n%s' % string_to_sign)
+        b64_hmac = self.sign_string(string_to_sign)
         auth_hdr = self._provider.auth_header
         headers['Authorization'] = ("%s %s:%s" %
                                     (auth_hdr,
@@ -136,8 +157,7 @@ class HmacAuthV2Handler(AuthHandler, HmacKeys):
     def add_auth(self, http_request, **kwargs):
         headers = http_request.headers
         if not headers.has_key('Date'):
-            headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
-                                            time.gmtime())
+            headers['Date'] = formatdate(usegmt=True)
 
         b64_hmac = self.sign_string(headers['Date'])
         auth_hdr = self._provider.auth_header
@@ -157,16 +177,92 @@ class HmacAuthV3Handler(AuthHandler, HmacKeys):
     def add_auth(self, http_request, **kwargs):
         headers = http_request.headers
         if not headers.has_key('Date'):
-            headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
-                                            time.gmtime())
+            headers['Date'] = formatdate(usegmt=True)
 
         b64_hmac = self.sign_string(headers['Date'])
         s = "AWS3-HTTPS AWSAccessKeyId=%s," % self._provider.access_key
         s += "Algorithm=%s,Signature=%s" % (self.algorithm(), b64_hmac)
         headers['X-Amzn-Authorization'] = s
 
+class HmacAuthV3HTTPHandler(AuthHandler, HmacKeys):
+    """
+    Implements the new Version 3 HMAC authorization used by DynamoDB.
+    """
+    
+    capability = ['hmac-v3-http']
+    
+    def __init__(self, host, config, provider):
+        AuthHandler.__init__(self, host, config, provider)
+        HmacKeys.__init__(self, host, config, provider)
+
+    def headers_to_sign(self, http_request):
+        """
+        Select the headers from the request that need to be included
+        in the StringToSign.
+        """
+        headers_to_sign = {}
+        headers_to_sign = {'Host' : self.host}
+        for name, value in http_request.headers.items():
+            lname = name.lower()
+            if lname.startswith('x-amz'):
+                headers_to_sign[name] = value
+        return headers_to_sign
+
+    def canonical_headers(self, headers_to_sign):
+        """
+        Return the headers that need to be included in the StringToSign
+        in their canonical form by converting all header keys to lower
+        case, sorting them in alphabetical order and then joining
+        them into a string, separated by newlines.
+        """
+        l = ['%s:%s'%(n.lower().strip(),
+                      headers_to_sign[n].strip()) for n in headers_to_sign]
+        l.sort()
+        return '\n'.join(l)
+        
+    def string_to_sign(self, http_request):
+        """
+        Return the canonical StringToSign as well as a dict
+        containing the original version of all headers that
+        were included in the StringToSign.
+        """
+        headers_to_sign = self.headers_to_sign(http_request)
+        canonical_headers = self.canonical_headers(headers_to_sign)
+        string_to_sign = '\n'.join([http_request.method,
+                                    http_request.path,
+                                    '',
+                                    canonical_headers,
+                                    '',
+                                    http_request.body])
+        return string_to_sign, headers_to_sign
+        
+    def add_auth(self, req, **kwargs):
+        """
+        Add AWS3 authentication to a request.
+
+        :type req: :class`boto.connection.HTTPRequest`
+        :param req: The HTTPRequest object.
+        """
+        # This could be a retry.  Make sure the previous
+        # authorization header is removed first.
+        if 'X-Amzn-Authorization' in req.headers:
+            del req.headers['X-Amzn-Authorization']
+        req.headers['X-Amz-Date'] = formatdate(usegmt=True)
+        if self._provider.security_token:
+            req.headers['X-Amz-Security-Token'] = self._provider.security_token
+        string_to_sign, headers_to_sign = self.string_to_sign(req)
+        boto.log.debug('StringToSign:\n%s' % string_to_sign)
+        hash_value = sha256(string_to_sign).digest()
+        b64_hmac = self.sign_string(hash_value)
+        s = "AWS3 AWSAccessKeyId=%s," % self._provider.access_key
+        s += "Algorithm=%s," % self.algorithm()
+        s += "SignedHeaders=%s," % ';'.join(headers_to_sign)
+        s += "Signature=%s" % b64_hmac
+        req.headers['X-Amzn-Authorization'] = s
+
 class QuerySignatureHelper(HmacKeys):
-    """Helper for Query signature based Auth handler.
+    """
+    Helper for Query signature based Auth handler.
 
     Concrete sub class need to implement _calc_sigature method.
     """
@@ -183,13 +279,15 @@ class QuerySignatureHelper(HmacKeys):
         boto.log.debug('query_string: %s Signature: %s' % (qs, signature))
         if http_request.method == 'POST':
             headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
-            http_request.body = qs + '&Signature=' + urllib.quote(signature)
+            http_request.body = qs + '&Signature=' + urllib.quote_plus(signature)
+            http_request.headers['Content-Length'] = str(len(http_request.body))
         else:
             http_request.body = ''
             # if this is a retried request, the qs from the previous try will
             # already be there, we need to get rid of that and rebuild it
             http_request.path = http_request.path.split('?')[0]
-            http_request.path = (http_request.path + '?' + qs + '&Signature=' + urllib.quote(signature))
+            http_request.path = (http_request.path + '?' + qs +
+                                 '&Signature=' + urllib.quote_plus(signature))
 
 class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler):
     """Provides Signature V0 Signing"""
@@ -238,7 +336,7 @@ class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler):
 
     SignatureVersion = 2
     capability = ['sign-v2', 'ec2', 'ec2', 'emr', 'fps', 'ecs',
-                  'sdb', 'iam', 'rds', 'sns', 'sqs']
+                  'sdb', 'iam', 'rds', 'sns', 'sqs', 'cloudformation']
 
     def _calc_signature(self, params, verb, path, server_name):
         boto.log.debug('using _calc_signature_2')
@@ -249,6 +347,8 @@ class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler):
         else:
             hmac = self._hmac.copy()
             params['SignatureMethod'] = 'HmacSHA1'
+        if self._provider.security_token:
+            params['SecurityToken'] = self._provider.security_token
         keys = params.keys()
         keys.sort()
         pairs = []
@@ -314,7 +414,10 @@ def get_auth_handler(host, config, provider, requested_capability=None):
         # on the wrong account.
         names = [handler.__class__.__name__ for handler in ready_handlers]
         raise boto.exception.TooManyAuthHandlerReadyToAuthenticate(
-               '%d AuthHandlers ready to authenticate, '
-               'only 1 expected: %s' % (len(names), str(names)))
+               '%d AuthHandlers %s ready to authenticate for requested_capability '
+               '%s, only 1 expected. This happens if you import multiple '
+               'pluging.Plugin implementations that declare support for the '
+               'requested_capability.' % (len(names), str(names),
+               requested_capability))
 
     return ready_handlers[0]