third_party/gsutil/boto/boto/s3/key.py - Issue 10199002: Upgrade gsutil to 3.4

Unified Diff: third_party/gsutil/boto/boto/s3/key.py

Issue 10199002: Upgrade gsutil to 3.4 (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart

Patch Set: Addressed comments Created 8 years, 8 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

Index: third_party/gsutil/boto/boto/s3/key.py

diff --git a/third_party/gsutil/20110627/boto/boto/s3/key.py b/third_party/gsutil/boto/boto/s3/key.py

similarity index 69%

rename from third_party/gsutil/20110627/boto/boto/s3/key.py

rename to third_party/gsutil/boto/boto/s3/key.py

index 2f716c4338ded2a311018dd494bd0facb9aac715..e8da98f34ff61301e05ad8285ecb59ae43965166 100644

--- a/third_party/gsutil/20110627/boto/boto/s3/key.py

+++ b/third_party/gsutil/boto/boto/s3/key.py

@@ -1,4 +1,5 @@

# Permission is hereby granted, free of charge, to any person obtaining a

# copy of this software and associated documentation files (the

@@ -14,7 +15,7 @@

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS

# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-

# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT

-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,

+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,

# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS

# IN THE SOFTWARE.

@@ -25,11 +26,14 @@ import re

import rfc822

import StringIO

import base64

+import math

+import urllib

import boto.utils

from boto.exception import BotoClientError

from boto.provider import Provider

from boto.s3.user import User

from boto import UserAgent

+from boto.utils import compute_md5

try:

from hashlib import md5

except ImportError:

@@ -49,8 +53,10 @@ class Key(object):

self.cache_control = None

self.content_type = self.DefaultContentType

self.content_encoding = None

+ self.content_disposition = None

self.filename = None

self.etag = None

+ self.is_latest = False

self.last_modified = None

self.owner = None

self.storage_class = 'STANDARD'

@@ -63,6 +69,7 @@ class Key(object):

self.version_id = None

self.source_version_id = None

self.delete_marker = False

+ self.encrypted = None

def __repr__(self):

if self.bucket:

@@ -92,7 +99,7 @@ class Key(object):

if self.bucket.connection:

provider = self.bucket.connection.provider

return provider

def get_md5_from_hexdigest(self, md5_hexdigest):

"""

A utility function to create the 2-tuple (md5hexdigest, base64md5)

@@ -104,7 +111,14 @@ class Key(object):

if base64md5[-1] == '\n':

base64md5 = base64md5[0:-1]

return (md5_hexdigest, base64md5)

+ def handle_encryption_headers(self, resp):

+ provider = self.bucket.connection.provider

+ if provider.server_side_encryption_header:

+ self.encrypted = resp.getheader(provider.server_side_encryption_header, None)

+ else:

+ self.encrypted = None

def handle_version_headers(self, resp, force=False):

provider = self.bucket.connection.provider

# If the Key object already has a version_id attribute value, it

@@ -121,17 +135,17 @@ class Key(object):

else:

self.delete_marker = False

- def open_read(self, headers=None, query_args=None,

+ def open_read(self, headers=None, query_args='',

override_num_retries=None, response_headers=None):

"""

Open this key for reading

:type headers: dict

:param headers: Headers to pass in the web request

:type query_args: string

:param query_args: Arguments to pass in the query string (ie, 'torrent')

:type override_num_retries: int

:param override_num_retries: If not None will override configured

num_retries parameter for underlying GET.

@@ -144,7 +158,7 @@ class Key(object):

"""

if self.resp == None:

self.mode = 'r'

provider = self.bucket.connection.provider

self.resp = self.bucket.connection.make_request(

'GET', self.bucket.name, self.name, headers,

@@ -177,13 +191,16 @@ class Key(object):

self.last_modified = value

elif name.lower() == 'cache-control':

self.cache_control = value

+ elif name.lower() == 'content-disposition':

+ self.content_disposition = value

self.handle_version_headers(self.resp)

+ self.handle_encryption_headers(self.resp)

def open_write(self, headers=None, override_num_retries=None):

"""

- Open this key for writing.

+ Open this key for writing.

Not yet implemented

:type headers: dict

:param headers: Headers to pass in the write request

@@ -213,7 +230,7 @@ class Key(object):

self.resp = None

self.mode = None

self.closed = True

def next(self):

"""

By providing a next method, the key object supports use as an iterator.

@@ -232,10 +249,11 @@ class Key(object):

return data

def read(self, size=0):

- if size == 0:

- size = self.BufferSize

self.open_read()

- data = self.resp.read(size)

+ if size == 0:

+ data = self.resp.read()

+ else:

+ data = self.resp.read(size)

if not data:

self.close()

return data

@@ -259,7 +277,7 @@ class Key(object):

:param dst_bucket: The name of a destination bucket. If not

provided the current bucket of the key

will be used.

"""

if new_storage_class == 'STANDARD':

return self.copy(self.bucket.name, self.name,

@@ -272,7 +290,8 @@ class Key(object):

new_storage_class)

def copy(self, dst_bucket, dst_key, metadata=None,

- reduced_redundancy=False, preserve_acl=False):

+ reduced_redundancy=False, preserve_acl=False,

+ encrypt_key=False):

"""

Copy this Key to another bucket.

@@ -281,7 +300,7 @@ class Key(object):

:type dst_key: string

:param dst_key: The name of the destination key

:type metadata: dict

:param metadata: Metadata to be associated with new key.

If metadata is supplied, it will replace the

@@ -312,6 +331,12 @@ class Key(object):

of False will be significantly more

efficient.

+ :type encrypt_key: bool

+ :param encrypt_key: If True, the new copy of the object will

+ be encrypted on the server-side by S3 and

+ will be stored in an encrypted form while

+ at rest in S3.

:rtype: :class:`boto.s3.key.Key` or subclass

:returns: An instance of the newly created key object

"""

@@ -323,7 +348,8 @@ class Key(object):

return dst_bucket.copy_key(dst_key, self.bucket.name,

self.name, metadata,

storage_class=storage_class,

- preserve_acl=preserve_acl)

+ preserve_acl=preserve_acl,

+ encrypt_key=encrypt_key)

def startElement(self, name, attrs, connection):

if name == 'Owner':

@@ -334,9 +360,14 @@ class Key(object):

def endElement(self, name, value, connection):

if name == 'Key':

- self.name = value.encode('utf-8')

+ self.name = value

elif name == 'ETag':

self.etag = value

+ elif name == 'IsLatest':

+ if value == 'true':

+ self.is_latest = True

+ else:

+ self.is_latest = False

elif name == 'LastModified':

self.last_modified = value

elif name == 'Size':

@@ -353,7 +384,7 @@ class Key(object):

def exists(self):

"""

Returns True if the key exists

:rtype: bool

:return: Whether the key exists on S3

"""

@@ -373,7 +404,7 @@ class Key(object):

def update_metadata(self, d):

self.metadata.update(d)

# convenience methods for setting/getting ACL

def set_acl(self, acl_str, headers=None):

if self.bucket != None:

@@ -393,28 +424,29 @@ class Key(object):

def set_canned_acl(self, acl_str, headers=None):

return self.bucket.set_canned_acl(acl_str, self.name, headers)

def make_public(self, headers=None):

return self.bucket.set_canned_acl('public-read', self.name, headers)

def generate_url(self, expires_in, method='GET', headers=None,

- query_auth=True, force_http=False, response_headers=None):

+ query_auth=True, force_http=False, response_headers=None,

+ expires_in_absolute=False):

"""

Generate a URL to access this key.

:type expires_in: int

:param expires_in: How long the url is valid for, in seconds

:type method: string

:param method: The method to use for retrieving the file

(default is GET)

:type headers: dict

:param headers: Any headers to pass along in the request

:type query_auth: bool

- :param query_auth:

+ :param query_auth:

:rtype: string

:return: The URL to access the key

"""

@@ -422,18 +454,25 @@ class Key(object):

self.bucket.name, self.name,

headers, query_auth,

force_http,

- response_headers)

+ response_headers,

+ expires_in_absolute)

- def send_file(self, fp, headers=None, cb=None, num_cb=10, query_args=None):

+ def send_file(self, fp, headers=None, cb=None, num_cb=10,

+ query_args=None, chunked_transfer=False, size=None):

"""

Upload a file to a key into a bucket on S3.

:type fp: file

- :param fp: The file pointer to upload

+ :param fp: The file pointer to upload. The file pointer must point

+ point at the offset from which you wish to upload.

+ ie. if uploading the full file, it should point at the

+ start of the file. Normally when a file is opened for

+ reading, the fp will point at the first byte. See the

+ bytes parameter below for more info.

:type headers: dict

:param headers: The headers to pass along with the PUT request

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -441,7 +480,7 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type num_cb: int

:param num_cb: (optional) If a callback is specified with the cb

parameter this parameter determines the granularity

@@ -449,16 +488,45 @@ class Key(object):

times the callback will be called during the file

transfer. Providing a negative integer will cause

your callback to be called with each buffer read.

+ :type size: int

+ :param size: (optional) The Maximum number of bytes to read from

+ the file pointer (fp). This is useful when uploading

+ a file in multiple parts where you are splitting the

+ file up into different ranges to be uploaded. If not

+ specified, the default behaviour is to read all bytes

+ from the file pointer. Less bytes may be available.

"""

provider = self.bucket.connection.provider

+ try:

+ spos = fp.tell()

+ except IOError:

+ spos = None

+ self.read_from_stream = False

def sender(http_conn, method, path, data, headers):

+ # This function is called repeatedly for temporary retries

+ # so we must be sure the file pointer is pointing at the

+ # start of the data.

+ if spos is not None and spos != fp.tell():

+ fp.seek(spos)

+ elif spos is None and self.read_from_stream:

+ # if seek is not supported, and we've read from this

+ # stream already, then we need to abort retries to

+ # avoid setting bad data.

+ raise provider.storage_data_error(

+ 'Cannot retry failed request. fp does not support seeking.')

http_conn.putrequest(method, path)

for key in headers:

http_conn.putheader(key, headers[key])

http_conn.endheaders()

- fp.seek(0)

+ if chunked_transfer and not self.base64md5:

+ # MD5 for the stream has to be calculated on the fly.

+ m = md5()

+ else:

+ m = None

save_debug = self.bucket.connection.debug

self.bucket.connection.debug = 0

# If the debuglevel < 3 we don't want to show connection

@@ -467,35 +535,81 @@ class Key(object):

# Use the getattr approach to allow this to work in AppEngine.

if getattr(http_conn, 'debuglevel', 0) < 3:

http_conn.set_debuglevel(0)

+ data_len = 0

if cb:

- if num_cb > 2:

- cb_count = self.size / self.BufferSize / (num_cb-2)

+ if size:

+ cb_size = size

+ elif self.size:

+ cb_size = self.size

+ else:

+ cb_size = 0

+ if chunked_transfer and cb_size == 0:

+ # For chunked Transfer, we call the cb for every 1MB

+ # of data transferred, except when we know size.

+ cb_count = (1024 * 1024)/self.BufferSize

+ elif num_cb > 1:

+ cb_count = int(math.ceil(cb_size/self.BufferSize/(num_cb-1.0)))

elif num_cb < 0:

cb_count = -1

else:

cb_count = 0

- i = total_bytes = 0

- cb(total_bytes, self.size)

- l = fp.read(self.BufferSize)

- while len(l) > 0:

- http_conn.send(l)

+ i = 0

+ cb(data_len, cb_size)

+ bytes_togo = size

+ if bytes_togo and bytes_togo < self.BufferSize:

+ chunk = fp.read(bytes_togo)

+ else:

+ chunk = fp.read(self.BufferSize)

+ if spos is None:

+ # read at least something from a non-seekable fp.

+ self.read_from_stream = True

+ while chunk:

+ chunk_len = len(chunk)

+ data_len += chunk_len

+ if chunked_transfer:

+ http_conn.send('%x;\r\n' % chunk_len)

+ http_conn.send(chunk)

+ http_conn.send('\r\n')

+ else:

+ http_conn.send(chunk)

+ if m:

+ m.update(chunk)

+ if bytes_togo:

+ bytes_togo -= chunk_len

+ if bytes_togo <= 0:

+ break

if cb:

- total_bytes += len(l)

i += 1

if i == cb_count or cb_count == -1:

- cb(total_bytes, self.size)

+ cb(data_len, cb_size)

i = 0

- l = fp.read(self.BufferSize)

- if cb:

- cb(total_bytes, self.size)

+ if bytes_togo and bytes_togo < self.BufferSize:

+ chunk = fp.read(bytes_togo)

+ else:

+ chunk = fp.read(self.BufferSize)

+ self.size = data_len

+ if chunked_transfer:

+ http_conn.send('0\r\n')

+ if m:

+ # Use the chunked trailer for the digest

+ hd = m.hexdigest()

+ self.md5, self.base64md5 = self.get_md5_from_hexdigest(hd)

+ # http_conn.send("Content-MD5: %s\r\n" % self.base64md5)

+ http_conn.send('\r\n')

+ if cb and (cb_count <= 1 or i > 0) and data_len > 0:

+ cb(data_len, cb_size)

response = http_conn.getresponse()

body = response.read()

- fp.seek(0)

http_conn.set_debuglevel(save_debug)

self.bucket.connection.debug = save_debug

- if response.status == 500 or response.status == 503 or \

- response.getheader('location'):

- # we'll try again

+ if ((response.status == 500 or response.status == 503 or

+ response.getheader('location')) and not chunked_transfer):

+ # we'll try again.

return response

elif response.status >= 200 and response.status <= 299:

self.etag = response.getheader('etag')

@@ -512,13 +626,20 @@ class Key(object):

else:

headers = headers.copy()

headers['User-Agent'] = UserAgent

- headers['Content-MD5'] = self.base64md5

if self.storage_class != 'STANDARD':

headers[provider.storage_class_header] = self.storage_class

if headers.has_key('Content-Encoding'):

self.content_encoding = headers['Content-Encoding']

if headers.has_key('Content-Type'):

- self.content_type = headers['Content-Type']

+ # Some use cases need to suppress sending of the Content-Type

+ # header and depend on the receiving server to set the content

+ # type. This can be achieved by setting headers['Content-Type']

+ # to None when calling this method.

+ if headers['Content-Type'] is None:

+ # Delete null Content-Type value to skip sending that header.

+ del headers['Content-Type']

+ else:

+ self.content_type = headers['Content-Type']

elif self.path:

self.content_type = mimetypes.guess_type(self.path)[0]

if self.content_type == None:

@@ -526,7 +647,14 @@ class Key(object):

headers['Content-Type'] = self.content_type

else:

headers['Content-Type'] = self.content_type

- headers['Content-Length'] = str(self.size)

+ if self.base64md5:

+ headers['Content-MD5'] = self.base64md5

+ if chunked_transfer:

+ headers['Transfer-Encoding'] = 'chunked'

+ #if not self.base64md5:

+ # headers['Trailer'] = "Content-MD5"

+ else:

+ headers['Content-Length'] = str(self.size)

headers['Expect'] = '100-Continue'

headers = boto.utils.merge_meta(headers, self.metadata, provider)

resp = self.bucket.connection.make_request('PUT', self.bucket.name,

@@ -535,43 +663,134 @@ class Key(object):

query_args=query_args)

self.handle_version_headers(resp, force=True)

- def compute_md5(self, fp):

+ def compute_md5(self, fp, size=None):

"""

:type fp: file

:param fp: File pointer to the file to MD5 hash. The file pointer

- will be reset to the beginning of the file before the

+ will be reset to the same position before the

method returns.

+ :type size: int

+ :param size: (optional) The Maximum number of bytes to read from

+ the file pointer (fp). This is useful when uploading

+ a file in multiple parts where the file is being

+ split inplace into different parts. Less bytes may

+ be available.

:rtype: tuple

:return: A tuple containing the hex digest version of the MD5 hash

as the first element and the base64 encoded version of the

plain digest as the second element.

"""

- m = md5()

- fp.seek(0)

- s = fp.read(self.BufferSize)

- while s:

- m.update(s)

- s = fp.read(self.BufferSize)

- hex_md5 = m.hexdigest()

- base64md5 = base64.encodestring(m.digest())

- if base64md5[-1] == '\n':

- base64md5 = base64md5[0:-1]

- self.size = fp.tell()

- fp.seek(0)

- return (hex_md5, base64md5)

+ tup = compute_md5(fp, size=size)

+ # Returned values are MD5 hash, base64 encoded MD5 hash, and data size.

+ # The internal implementation of compute_md5() needs to return the

+ # data size but we don't want to return that value to the external

+ # caller because it changes the class interface (i.e. it might

+ # break some code) so we consume the third tuple value here and

+ # return the remainder of the tuple to the caller, thereby preserving

+ # the existing interface.

+ self.size = tup[2]

+ return tup[0:2]

+ def set_contents_from_stream(self, fp, headers=None, replace=True,

+ cb=None, num_cb=10, policy=None,

+ reduced_redundancy=False, query_args=None,

+ size=None):

+ """

+ Store an object using the name of the Key object as the key in

+ cloud and the contents of the data stream pointed to by 'fp' as

+ the contents.

+ The stream object is not seekable and total size is not known.

+ This has the implication that we can't specify the Content-Size and

+ Content-MD5 in the header. So for huge uploads, the delay in calculating

+ MD5 is avoided but with a penalty of inability to verify the integrity

+ of the uploaded data.

+ :type fp: file

+ :param fp: the file whose contents are to be uploaded

+ :type headers: dict

+ :param headers: additional HTTP headers to be sent with the PUT request.

+ :type replace: bool

+ :param replace: If this parameter is False, the method will first check

+ to see if an object exists in the bucket with the same key. If it

+ does, it won't overwrite it. The default value is True which will

+ overwrite the object.

+ :type cb: function

+ :param cb: a callback function that will be called to report

+ progress on the upload. The callback should accept two integer

+ parameters, the first representing the number of bytes that have

+ been successfully transmitted to GS and the second representing the

+ total number of bytes that need to be transmitted.

+ :type num_cb: int

+ :param num_cb: (optional) If a callback is specified with the cb

+ parameter, this parameter determines the granularity of the callback

+ by defining the maximum number of times the callback will be called

+ during the file transfer.

+ :type policy: :class:`boto.gs.acl.CannedACLStrings`

+ :param policy: A canned ACL policy that will be applied to the new key

+ in GS.

+ :type reduced_redundancy: bool

+ :param reduced_redundancy: If True, this will set the storage

+ class of the new Key to be

+ REDUCED_REDUNDANCY. The Reduced Redundancy

+ Storage (RRS) feature of S3, provides lower

+ redundancy at lower storage cost.

+ :type size: int

+ :param size: (optional) The Maximum number of bytes to read from

+ the file pointer (fp). This is useful when uploading

+ a file in multiple parts where you are splitting the

+ file up into different ranges to be uploaded. If not

+ specified, the default behaviour is to read all bytes

+ from the file pointer. Less bytes may be available.

+ """

+ provider = self.bucket.connection.provider

+ if not provider.supports_chunked_transfer():

+ raise BotoClientError('%s does not support chunked transfer'

+ % provider.get_provider_name())

+ # Name of the Object should be specified explicitly for Streams.

+ if not self.name or self.name == '':

+ raise BotoClientError('Cannot determine the destination '

+ 'object name for the given stream')

+ if headers is None:

+ headers = {}

+ if policy:

+ headers[provider.acl_header] = policy

+ if reduced_redundancy:

+ self.storage_class = 'REDUCED_REDUNDANCY'

+ if provider.storage_class_header:

+ headers[provider.storage_class_header] = self.storage_class

+ if self.bucket != None:

+ if not replace:

+ if self.bucket.lookup(self.name):

+ return

+ self.send_file(fp, headers, cb, num_cb, query_args,

+ chunked_transfer=True, size=size)

def set_contents_from_file(self, fp, headers=None, replace=True,

cb=None, num_cb=10, policy=None, md5=None,

- reduced_redundancy=False, query_args=None):

+ reduced_redundancy=False, query_args=None,

+ encrypt_key=False, size=None, rewind=False):

"""

Store an object in S3 using the name of the Key object as the

key in S3 and the contents of the file pointed to by 'fp' as the

- contents.

+ contents. The data is read from 'fp' from its current position until

+ 'size' bytes have been read or EOF.

:type fp: file

:param fp: the file whose contents to upload

:type headers: dict

:param headers: Additional HTTP headers that will be sent with

the PUT request.

@@ -582,7 +801,7 @@ class Key(object):

bucket with the same key. If it does, it won't

overwrite it. The default value is True which will

overwrite the object.

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -590,7 +809,7 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with the cb

parameter this parameter determines the granularity

@@ -601,7 +820,7 @@ class Key(object):

:type policy: :class:`boto.s3.acl.CannedACLStrings`

:param policy: A canned ACL policy that will be applied to the

new key in S3.

:type md5: A tuple containing the hexdigest version of the MD5

checksum of the file as the first element and the

Base64-encoded version of the plain checksum as the

@@ -611,7 +830,7 @@ class Key(object):

to upload, it's silly to have to do it twice so this

param, if present, will be used as the MD5 values of

the file. Otherwise, the checksum will be computed.

:type reduced_redundancy: bool

:param reduced_redundancy: If True, this will set the storage

class of the new Key to be

@@ -619,12 +838,53 @@ class Key(object):

Storage (RRS) feature of S3, provides lower

redundancy at lower storage cost.

+ :type encrypt_key: bool

+ :param encrypt_key: If True, the new copy of the object will

+ be encrypted on the server-side by S3 and

+ will be stored in an encrypted form while

+ at rest in S3.

+ :type size: int

+ :param size: (optional) The Maximum number of bytes to read from

+ the file pointer (fp). This is useful when uploading

+ a file in multiple parts where you are splitting the

+ file up into different ranges to be uploaded. If not

+ specified, the default behaviour is to read all bytes

+ from the file pointer. Less bytes may be available.

+ :type rewind: bool

+ :param rewind: (optional) If True, the file pointer (fp) will be

+ rewound to the start before any bytes are read from

+ it. The default behaviour is False which reads from

+ the current position of the file pointer (fp).

+ :rtype: int

+ :return: The number of bytes written to the key.

"""

provider = self.bucket.connection.provider

- if headers is None:

- headers = {}

+ headers = headers or {}

if policy:

headers[provider.acl_header] = policy

+ if encrypt_key:

+ headers[provider.server_side_encryption_header] = 'AES256'

+ if rewind:

+ # caller requests reading from beginning of fp.

+ fp.seek(0, os.SEEK_SET)

+ else:

+ spos = fp.tell()

+ fp.seek(0, os.SEEK_END)

+ if fp.tell() == spos:

+ fp.seek(0, os.SEEK_SET)

+ if fp.tell() != spos:

+ # Raise an exception as this is likely a programming error

+ # whereby there is data before the fp but nothing after it.

+ fp.seek(spos)

+ raise AttributeError(

+ 'fp is at EOF. Use rewind option or seek() to data start.')

+ # seek back to the correct position.

+ fp.seek(spos)

if reduced_redundancy:

self.storage_class = 'REDUCED_REDUNDANCY'

if provider.storage_class_header:

@@ -633,44 +893,67 @@ class Key(object):

# What if different providers provide different classes?

if hasattr(fp, 'name'):

self.path = fp.name

if self.bucket != None:

- if not md5:

- md5 = self.compute_md5(fp)

+ if not md5 and provider.supports_chunked_transfer():

+ # defer md5 calculation to on the fly and

+ # we don't know anything about size yet.

+ chunked_transfer = True

+ self.size = None

else:

- # even if md5 is provided, still need to set size of content

- fp.seek(0, 2)

- self.size = fp.tell()

- fp.seek(0)

- self.md5 = md5[0]

- self.base64md5 = md5[1]

+ chunked_transfer = False

+ if not md5:

+ # compute_md5() and also set self.size to actual

+ # size of the bytes read computing the md5.

+ md5 = self.compute_md5(fp, size)

+ # adjust size if required

+ size = self.size

+ elif size:

+ self.size = size

+ else:

+ # If md5 is provided, still need to size so

+ # calculate based on bytes to end of content

+ spos = fp.tell()

+ fp.seek(0, os.SEEK_END)

+ self.size = fp.tell() - spos

+ fp.seek(spos)

+ size = self.size

+ self.md5 = md5[0]

+ self.base64md5 = md5[1]

if self.name == None:

self.name = self.md5

if not replace:

- k = self.bucket.lookup(self.name)

- if k:

+ if self.bucket.lookup(self.name):

return

- self.send_file(fp, headers, cb, num_cb, query_args)

+ self.send_file(fp, headers=headers, cb=cb, num_cb=num_cb,

+ query_args=query_args, chunked_transfer=chunked_transfer,

+ size=size)

+ # return number of bytes written.

+ return self.size

def set_contents_from_filename(self, filename, headers=None, replace=True,

cb=None, num_cb=10, policy=None, md5=None,

- reduced_redundancy=False):

+ reduced_redundancy=False,

+ encrypt_key=False):

"""

Store an object in S3 using the name of the Key object as the

key in S3 and the contents of the file named by 'filename'.

See set_contents_from_file method for details about the

parameters.

:type filename: string

:param filename: The name of the file that you want to put onto S3

:type headers: dict

:param headers: Additional headers to pass along with the

request to AWS.

:type replace: bool

:param replace: If True, replaces the contents of the file

if it already exists.

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -678,18 +961,18 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with

the cb parameter this parameter determines the

granularity of the callback by defining

the maximum number of times the callback will

- be called during the file transfer.

+ be called during the file transfer.

:type policy: :class:`boto.s3.acl.CannedACLStrings`

:param policy: A canned ACL policy that will be applied to the

new key in S3.

:type md5: A tuple containing the hexdigest version of the MD5

checksum of the file as the first element and the

Base64-encoded version of the plain checksum as the

@@ -699,36 +982,43 @@ class Key(object):

to upload, it's silly to have to do it twice so this

param, if present, will be used as the MD5 values

of the file. Otherwise, the checksum will be computed.

:type reduced_redundancy: bool

:param reduced_redundancy: If True, this will set the storage

class of the new Key to be

REDUCED_REDUNDANCY. The Reduced Redundancy

Storage (RRS) feature of S3, provides lower

redundancy at lower storage cost.

+ :type encrypt_key: bool

+ :param encrypt_key: If True, the new copy of the object will

+ be encrypted on the server-side by S3 and

+ will be stored in an encrypted form while

+ at rest in S3.

"""

fp = open(filename, 'rb')

self.set_contents_from_file(fp, headers, replace, cb, num_cb,

- policy, md5, reduced_redundancy)

+ policy, md5, reduced_redundancy,

+ encrypt_key=encrypt_key)

fp.close()

def set_contents_from_string(self, s, headers=None, replace=True,

cb=None, num_cb=10, policy=None, md5=None,

- reduced_redundancy=False):

+ reduced_redundancy=False,

+ encrypt_key=False):

"""

Store an object in S3 using the name of the Key object as the

key in S3 and the string 's' as the contents.

See set_contents_from_file method for details about the

parameters.

:type headers: dict

:param headers: Additional headers to pass along with the

request to AWS.

:type replace: bool

:param replace: If True, replaces the contents of the file if

it already exists.

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -736,18 +1026,18 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with

the cb parameter this parameter determines the

granularity of the callback by defining

the maximum number of times the callback will

- be called during the file transfer.

+ be called during the file transfer.

:type policy: :class:`boto.s3.acl.CannedACLStrings`

:param policy: A canned ACL policy that will be applied to the

new key in S3.

:type md5: A tuple containing the hexdigest version of the MD5

checksum of the file as the first element and the

Base64-encoded version of the plain checksum as the

@@ -757,19 +1047,25 @@ class Key(object):

to upload, it's silly to have to do it twice so this

param, if present, will be used as the MD5 values

of the file. Otherwise, the checksum will be computed.

:type reduced_redundancy: bool

:param reduced_redundancy: If True, this will set the storage

class of the new Key to be

REDUCED_REDUNDANCY. The Reduced Redundancy

Storage (RRS) feature of S3, provides lower

redundancy at lower storage cost.

+ :type encrypt_key: bool

+ :param encrypt_key: If True, the new copy of the object will

+ be encrypted on the server-side by S3 and

+ will be stored in an encrypted form while

+ at rest in S3.

"""

if isinstance(s, unicode):

s = s.encode("utf-8")

fp = StringIO.StringIO(s)

r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,

- policy, md5, reduced_redundancy)

+ policy, md5, reduced_redundancy,

+ encrypt_key=encrypt_key)

fp.close()

return r

@@ -778,13 +1074,13 @@ class Key(object):

response_headers=None):

"""

Retrieves a file from an S3 Key

:type fp: file

:param fp: File pointer to put the data into

:type headers: string

:param: headers to send when retrieving the files

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -792,14 +1088,14 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with

the cb parameter this parameter determines the

granularity of the callback by defining

the maximum number of times the callback will

- be called during the file transfer.

+ be called during the file transfer.

:type torrent: bool

:param torrent: Flag for whether to get a torrent for the file

@@ -813,22 +1109,16 @@ class Key(object):

the stored object in the response.

See http://goo.gl/EWOPb for details.

"""

- if cb:

- if num_cb > 2:

- cb_count = self.size / self.BufferSize / (num_cb-2)

- elif num_cb < 0:

- cb_count = -1

- else:

- cb_count = 0

- i = total_bytes = 0

- cb(total_bytes, self.size)

save_debug = self.bucket.connection.debug

if self.bucket.connection.debug == 1:

self.bucket.connection.debug = 0

query_args = []

if torrent:

query_args.append('torrent')

+ m = None

+ else:

+ m = md5()

# If a version_id is passed in, use that. If not, check to see

# if the Key object has an explicit version_id and, if so, use that.

# Otherwise, don't pass a version_id query param.

@@ -838,33 +1128,60 @@ class Key(object):

query_args.append('versionId=%s' % version_id)

if response_headers:

for key in response_headers:

- query_args.append('%s=%s' % (key, response_headers[key]))

+ query_args.append('%s=%s' % (key, urllib.quote(response_headers[key])))

query_args = '&'.join(query_args)

self.open('r', headers, query_args=query_args,

override_num_retries=override_num_retries)

+ data_len = 0

+ if cb:

+ if self.size is None:

+ cb_size = 0

+ else:

+ cb_size = self.size

+ if self.size is None and num_cb != -1:

+ # If size is not available due to chunked transfer for example,

+ # we'll call the cb for every 1MB of data transferred.

+ cb_count = (1024 * 1024)/self.BufferSize

+ elif num_cb > 1:

+ cb_count = int(math.ceil(cb_size/self.BufferSize/(num_cb-1.0)))

+ elif num_cb < 0:

+ cb_count = -1

+ else:

+ cb_count = 0

+ i = 0

+ cb(data_len, cb_size)

for bytes in self:

fp.write(bytes)

+ data_len += len(bytes)

+ if m:

+ m.update(bytes)

if cb:

- total_bytes += len(bytes)

+ if cb_size > 0 and data_len >= cb_size:

+ break

i += 1

if i == cb_count or cb_count == -1:

- cb(total_bytes, self.size)

+ cb(data_len, cb_size)

i = 0

- if cb:

- cb(total_bytes, self.size)

+ if cb and (cb_count <= 1 or i > 0) and data_len > 0:

+ cb(data_len, cb_size)

+ if m:

+ self.md5 = m.hexdigest()

+ if self.size is None and not torrent and not headers.has_key("Range"):

+ self.size = data_len

self.close()

self.bucket.connection.debug = save_debug

def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10):

"""

Get a torrent file (see to get_file)

:type fp: file

:param fp: The file pointer of where to put the torrent

:type headers: dict

:param headers: Headers to be passed

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -872,17 +1189,17 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with

the cb parameter this parameter determines the

granularity of the callback by defining

the maximum number of times the callback will

- be called during the file transfer.

+ be called during the file transfer.

"""

return self.get_file(fp, headers, cb, num_cb, torrent=True)

def get_contents_to_file(self, fp, headers=None,

cb=None, num_cb=10,

torrent=False,

@@ -893,14 +1210,14 @@ class Key(object):

Retrieve an object from S3 using the name of the Key object as the

key in S3. Write the contents of the object to the file pointed

to by 'fp'.

:type fp: File -like object

:param fp:

:type headers: dict

:param headers: additional HTTP headers that will be sent with

the GET request.

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -908,14 +1225,14 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with

the cb parameter this parameter determines the

granularity of the callback by defining

the maximum number of times the callback will

- be called during the file transfer.

+ be called during the file transfer.

:type torrent: bool

:param torrent: If True, returns the contents of a torrent

file as a string.

@@ -951,13 +1268,13 @@ class Key(object):

key in S3. Store contents of the object to a file named by 'filename'.

See get_contents_to_file method for details about the

parameters.

:type filename: string

:param filename: The filename of where to put the file contents

:type headers: dict

:param headers: Any additional headers to send in the request

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -965,14 +1282,14 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with

the cb parameter this parameter determines the

granularity of the callback by defining

the maximum number of times the callback will

- be called during the file transfer.

+ be called during the file transfer.

:type torrent: bool

:param torrent: If True, returns the contents of a torrent file

as a string.

@@ -1011,10 +1328,10 @@ class Key(object):

key in S3. Return the contents of the object as a string.

See get_contents_to_file method for details about the

parameters.

:type headers: dict

:param headers: Any additional headers to send in the request

:type cb: function

:param cb: a callback function that will be called to report

progress on the upload. The callback should accept

@@ -1022,24 +1339,24 @@ class Key(object):

number of bytes that have been successfully

transmitted to S3 and the second representing the

size of the to be transmitted object.

:type cb: int

:param num_cb: (optional) If a callback is specified with

the cb parameter this parameter determines the

granularity of the callback by defining

the maximum number of times the callback will

- be called during the file transfer.

+ be called during the file transfer.

:type torrent: bool

:param torrent: If True, returns the contents of a torrent file

as a string.

:type response_headers: dict

:param response_headers: A dictionary containing HTTP headers/values

that will override any headers associated with

the stored object in the response.

See http://goo.gl/EWOPb for details.

:rtype: string

:returns: The contents of the file as a string

"""

@@ -1055,15 +1372,15 @@ class Key(object):

to a key. This method retrieves the current ACL, creates a new

grant based on the parameters passed in, adds that grant to the ACL

and then PUT's the new ACL back to S3.

:type permission: string

:param permission: The permission being granted. Should be one of:

(READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL).

:type email_address: string

:param email_address: The email address associated with the AWS

account your are granting the permission to.

:type recursive: boolean

:param recursive: A boolean value to controls whether the command

will apply the grant to all keys within the bucket

@@ -1077,30 +1394,27 @@ class Key(object):

policy.acl.add_email_grant(permission, email_address)

self.set_acl(policy, headers=headers)

- def add_user_grant(self, permission, user_id, headers=None):

+ def add_user_grant(self, permission, user_id, headers=None,

+ display_name=None):

"""

Convenience method that provides a quick way to add a canonical

user grant to a key. This method retrieves the current ACL,

creates a new grant based on the parameters passed in, adds that

grant to the ACL and then PUT's the new ACL back to S3.

:type permission: string

:param permission: The permission being granted. Should be one of:

(READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL).

:type user_id: string

:param user_id: The canonical user id associated with the AWS

account your are granting the permission to.

- :type recursive: boolean

- :param recursive: A boolean value to controls whether the command

- will apply the grant to all keys within the bucket

- or not. The default value is False. By passing a

- True value, the call will iterate through all keys

- in the bucket and apply the same grant to each key.

- CAUTION: If you have a lot of keys, this could take

- a long time!

+ :type display_name: string

+ :param display_name: An option string containing the user's

+ Display Name. Only required on Walrus.

"""

policy = self.get_acl()

- policy.acl.add_user_grant(permission, user_id)

+ policy.acl.add_user_grant(permission, user_id,

+ display_name=display_name)

self.set_acl(policy, headers=headers)

« no previous file with comments | « third_party/gsutil/boto/boto/s3/deletemarker.py ('k') | third_party/gsutil/boto/boto/s3/lifecycle.py » ('j') | no next file with comments »