Scripts to download files from google storage based on sha1 sums

third_party/gsutil/boto/dynamodb/table.py

+# Copyright (c) 2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.  All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+from boto.dynamodb.batch import BatchList
+from boto.dynamodb.schema import Schema
+from boto.dynamodb.item import Item
+from boto.dynamodb import exceptions as dynamodb_exceptions
+import time
+
+class TableBatchGenerator(object):
+    """
+    A low-level generator used to page through results from
+    batch_get_item operations.
+
+    :ivar consumed_units: An integer that holds the number of
+        ConsumedCapacityUnits accumulated thus far for this
+        generator.
+    """
+
+    def __init__(self, table, keys, attributes_to_get=None, consistent_read=False):
+        self.table = table
+        self.keys = keys
+        self.consumed_units = 0
+        self.attributes_to_get = attributes_to_get
+        self.consistent_read = consistent_read
+
+    def _queue_unprocessed(self, res):
+        if not u'UnprocessedKeys' in res:
+            return
+        if not self.table.name in res[u'UnprocessedKeys']:
+            return
+
+        keys = res[u'UnprocessedKeys'][self.table.name][u'Keys']
+
+        for key in keys:
+            h = key[u'HashKeyElement']
+            r = key[u'RangeKeyElement'] if u'RangeKeyElement' in key else None
+            self.keys.append((h, r))
+
+    def __iter__(self):
+        while self.keys:
+            # Build the next batch
+            batch = BatchList(self.table.layer2)
+            batch.add_batch(self.table, self.keys[:100], self.attributes_to_get)
+            res = batch.submit()
+
+            # parse the results
+            if not self.table.name in res[u'Responses']:
+                continue
+            self.consumed_units += res[u'Responses'][self.table.name][u'ConsumedCapacityUnits']
+            for elem in res[u'Responses'][self.table.name][u'Items']:
+                yield elem
+
+            # re-queue un processed keys
+            self.keys = self.keys[100:]
+            self._queue_unprocessed(res)
+
+
+class Table(object):
+    """
+    An Amazon DynamoDB table.
+
+    :ivar name: The name of the table.
+    :ivar create_time: The date and time that the table was created.
+    :ivar status: The current status of the table.  One of:
+        'ACTIVE', 'UPDATING', 'DELETING'.
+    :ivar schema: A :class:`boto.dynamodb.schema.Schema` object representing
+        the schema defined for the table.
+    :ivar item_count: The number of items in the table.  This value is
+        set only when the Table object is created or refreshed and
+        may not reflect the actual count.
+    :ivar size_bytes: Total size of the specified table, in bytes.
+        Amazon DynamoDB updates this value approximately every six hours.
+        Recent changes might not be reflected in this value.
+    :ivar read_units: The ReadCapacityUnits of the tables
+        Provisioned Throughput.
+    :ivar write_units: The WriteCapacityUnits of the tables
+        Provisioned Throughput.
+    :ivar schema: The Schema object associated with the table.
+    """
+
+    def __init__(self, layer2, response):
+        """
+
+        :type layer2: :class:`boto.dynamodb.layer2.Layer2`
+        :param layer2: A `Layer2` api object.
+
+        :type response: dict
+        :param response: The output of
+            `boto.dynamodb.layer1.Layer1.describe_table`.
+
+        """
+        self.layer2 = layer2
+        self._dict = {}
+        self.update_from_response(response)
+
+    def __repr__(self):
+        return 'Table(%s)' % self.name
+
+    @property
+    def name(self):
+        return self._dict['TableName']
+
+    @property
+    def create_time(self):
+        return self._dict['CreationDateTime']
+
+    @property
+    def status(self):
+        return self._dict['TableStatus']
+
+    @property
+    def item_count(self):
+        return self._dict.get('ItemCount', 0)
+
+    @property
+    def size_bytes(self):
+        return self._dict.get('TableSizeBytes', 0)
+
+    @property
+    def schema(self):
+        return self._schema
+
+    @property
+    def read_units(self):
+        return self._dict['ProvisionedThroughput']['ReadCapacityUnits']
+
+    @property
+    def write_units(self):
+        return self._dict['ProvisionedThroughput']['WriteCapacityUnits']
+
+    def update_from_response(self, response):
+        """
+        Update the state of the Table object based on the response
+        data received from Amazon DynamoDB.
+        """
+        if 'Table' in response:
+            self._dict.update(response['Table'])
+        elif 'TableDescription' in response:
+            self._dict.update(response['TableDescription'])
+        if 'KeySchema' in self._dict:
+            self._schema = Schema(self._dict['KeySchema'])
+
+    def refresh(self, wait_for_active=False, retry_seconds=5):
+        """
+        Refresh all of the fields of the Table object by calling
+        the underlying DescribeTable request.
+
+        :type wait_for_active: bool
+        :param wait_for_active: If True, this command will not return
+            until the table status, as returned from Amazon DynamoDB, is
+            'ACTIVE'.
+
+        :type retry_seconds: int
+        :param retry_seconds: If wait_for_active is True, this
+            parameter controls the number of seconds of delay between
+            calls to update_table in Amazon DynamoDB.  Default is 5 seconds.
+        """
+        done = False
+        while not done:
+            response = self.layer2.describe_table(self.name)
+            self.update_from_response(response)
+            if wait_for_active:
+                if self.status == 'ACTIVE':
+                    done = True
+                else:
+                    time.sleep(retry_seconds)
+            else:
+                done = True
+
+    def update_throughput(self, read_units, write_units):
+        """
+        Update the ProvisionedThroughput for the Amazon DynamoDB Table.
+
+        :type read_units: int
+        :param read_units: The new value for ReadCapacityUnits.
+
+        :type write_units: int
+        :param write_units: The new value for WriteCapacityUnits.
+        """
+        self.layer2.update_throughput(self, read_units, write_units)
+
+    def delete(self):
+        """
+        Delete this table and all items in it.  After calling this
+        the Table objects status attribute will be set to 'DELETING'.
+        """
+        self.layer2.delete_table(self)
+
+    def get_item(self, hash_key, range_key=None,
+                 attributes_to_get=None, consistent_read=False,
+                 item_class=Item):
+        """
+        Retrieve an existing item from the table.
+
+        :type hash_key: int|long|float|str|unicode
+        :param hash_key: The HashKey of the requested item.  The
+            type of the value must match the type defined in the
+            schema for the table.
+
+        :type range_key: int|long|float|str|unicode
+        :param range_key: The optional RangeKey of the requested item.
+            The type of the value must match the type defined in the
+            schema for the table.
+
+        :type attributes_to_get: list
+        :param attributes_to_get: A list of attribute names.
+            If supplied, only the specified attribute names will
+            be returned.  Otherwise, all attributes will be returned.
+
+        :type consistent_read: bool
+        :param consistent_read: If True, a consistent read
+            request is issued.  Otherwise, an eventually consistent
+            request is issued.
+
+        :type item_class: Class
+        :param item_class: Allows you to override the class used
+            to generate the items. This should be a subclass of
+            :class:`boto.dynamodb.item.Item`
+        """
+        return self.layer2.get_item(self, hash_key, range_key,
+                                    attributes_to_get, consistent_read,
+                                    item_class)
+    lookup = get_item
+
+    def has_item(self, hash_key, range_key=None, consistent_read=False):
+        """
+        Checks the table to see if the Item with the specified ``hash_key``
+        exists. This may save a tiny bit of time/bandwidth over a
+        straight :py:meth:`get_item` if you have no intention to touch
+        the data that is returned, since this method specifically tells
+        Amazon not to return anything but the Item's key.
+
+        :type hash_key: int|long|float|str|unicode
+        :param hash_key: The HashKey of the requested item.  The
+            type of the value must match the type defined in the
+            schema for the table.
+
+        :type range_key: int|long|float|str|unicode
+        :param range_key: The optional RangeKey of the requested item.
+            The type of the value must match the type defined in the
+            schema for the table.
+
+        :type consistent_read: bool
+        :param consistent_read: If True, a consistent read
+            request is issued.  Otherwise, an eventually consistent
+            request is issued.
+
+        :rtype: bool
+        :returns: ``True`` if the Item exists, ``False`` if not.
+        """
+        try:
+            # Attempt to get the key. If it can't be found, it'll raise
+            # an exception.
+            self.get_item(hash_key, range_key=range_key,
+                          # This minimizes the size of the response body.
+                          attributes_to_get=[hash_key],
+                          consistent_read=consistent_read)
+        except dynamodb_exceptions.DynamoDBKeyNotFoundError:
+            # Key doesn't exist.
+            return False
+        return True
+
+    def new_item(self, hash_key=None, range_key=None, attrs=None,
+                 item_class=Item):
+        """
+        Return an new, unsaved Item which can later be PUT to
+        Amazon DynamoDB.
+
+        This method has explicit (but optional) parameters for
+        the hash_key and range_key values of the item.  You can use
+        these explicit parameters when calling the method, such as::
+
+        >>> my_item = my_table.new_item(hash_key='a', range_key=1,
+                                        attrs={'key1': 'val1', 'key2': 'val2'})
+            >>> my_item
+            {u'bar': 1, u'foo': 'a', 'key1': 'val1', 'key2': 'val2'}
+
+        Or, if you prefer, you can simply put the hash_key and range_key
+        in the attrs dictionary itself, like this::
+
+            >>> attrs = {'foo': 'a', 'bar': 1, 'key1': 'val1', 'key2': 'val2'}
+            >>> my_item = my_table.new_item(attrs=attrs)
+            >>> my_item
+            {u'bar': 1, u'foo': 'a', 'key1': 'val1', 'key2': 'val2'}
+
+        The effect is the same.
+
+        .. note:
+           The explicit parameters take priority over the values in
+           the attrs dict.  So, if you have a hash_key or range_key
+           in the attrs dict and you also supply either or both using
+           the explicit parameters, the values in the attrs will be
+           ignored.
+
+        :type hash_key: int|long|float|str|unicode
+        :param hash_key: The HashKey of the new item.  The
+            type of the value must match the type defined in the
+            schema for the table.
+
+        :type range_key: int|long|float|str|unicode
+        :param range_key: The optional RangeKey of the new item.
+            The type of the value must match the type defined in the
+            schema for the table.
+
+        :type attrs: dict
+        :param attrs: A dictionary of key value pairs used to
+            populate the new item.
+
+        :type item_class: Class
+        :param item_class: Allows you to override the class used
+            to generate the items. This should be a subclass of
+            :class:`boto.dynamodb.item.Item`
+        """
+        return item_class(self, hash_key, range_key, attrs)
+
+    def query(self, hash_key, range_key_condition=None,
+              attributes_to_get=None, request_limit=None,
+              max_results=None, consistent_read=False,
+              scan_index_forward=True, exclusive_start_key=None,
+              item_class=Item):
+        """
+        Perform a query on the table.
+
+        :type hash_key: int|long|float|str|unicode
+        :param hash_key: The HashKey of the requested item.  The
+            type of the value must match the type defined in the
+            schema for the table.
+
+        :type range_key_condition: :class:`boto.dynamodb.condition.Condition`
+        :param range_key_condition: A Condition object.
+            Condition object can be one of the following types:
+
+            EQ|LE|LT|GE|GT|BEGINS_WITH|BETWEEN
+
+            The only condition which expects or will accept two
+            values is 'BETWEEN', otherwise a single value should
+            be passed to the Condition constructor.
+
+        :type attributes_to_get: list
+        :param attributes_to_get: A list of attribute names.
+            If supplied, only the specified attribute names will
+            be returned.  Otherwise, all attributes will be returned.
+
+        :type request_limit: int
+        :param request_limit: The maximum number of items to retrieve
+            from Amazon DynamoDB on each request.  You may want to set
+            a specific request_limit based on the provisioned throughput
+            of your table.  The default behavior is to retrieve as many
+            results as possible per request.
+
+        :type max_results: int
+        :param max_results: The maximum number of results that will
+            be retrieved from Amazon DynamoDB in total.  For example,
+            if you only wanted to see the first 100 results from the
+            query, regardless of how many were actually available, you
+            could set max_results to 100 and the generator returned
+            from the query method will only yeild 100 results max.
+
+        :type consistent_read: bool
+        :param consistent_read: If True, a consistent read
+            request is issued.  Otherwise, an eventually consistent
+            request is issued.
+
+        :type scan_index_forward: bool
+        :param scan_index_forward: Specified forward or backward
+            traversal of the index.  Default is forward (True).
+
+        :type exclusive_start_key: list or tuple
+        :param exclusive_start_key: Primary key of the item from
+            which to continue an earlier query.  This would be
+            provided as the LastEvaluatedKey in that query.
+
+        :type item_class: Class
+        :param item_class: Allows you to override the class used
+            to generate the items. This should be a subclass of
+            :class:`boto.dynamodb.item.Item`
+        """
+        return self.layer2.query(self, hash_key, range_key_condition,
+                                 attributes_to_get, request_limit,
+                                 max_results, consistent_read,
+                                 scan_index_forward, exclusive_start_key,
+                                 item_class=item_class)
+
+    def scan(self, scan_filter=None,
+             attributes_to_get=None, request_limit=None, max_results=None,
+             count=False, exclusive_start_key=None, item_class=Item):
+        """
+        Scan through this table, this is a very long
+        and expensive operation, and should be avoided if
+        at all possible.
+
+        :type scan_filter: A dict
+        :param scan_filter: A dictionary where the key is the
+            attribute name and the value is a
+            :class:`boto.dynamodb.condition.Condition` object.
+            Valid Condition objects include:
+
+             * EQ - equal (1)
+             * NE - not equal (1)
+             * LE - less than or equal (1)
+             * LT - less than (1)
+             * GE - greater than or equal (1)
+             * GT - greater than (1)
+             * NOT_NULL - attribute exists (0, use None)
+             * NULL - attribute does not exist (0, use None)
+             * CONTAINS - substring or value in list (1)
+             * NOT_CONTAINS - absence of substring or value in list (1)
+             * BEGINS_WITH - substring prefix (1)
+             * IN - exact match in list (N)
+             * BETWEEN - >= first value, <= second value (2)
+
+        :type attributes_to_get: list
+        :param attributes_to_get: A list of attribute names.
+            If supplied, only the specified attribute names will
+            be returned.  Otherwise, all attributes will be returned.
+
+        :type request_limit: int
+        :param request_limit: The maximum number of items to retrieve
+            from Amazon DynamoDB on each request.  You may want to set
+            a specific request_limit based on the provisioned throughput
+            of your table.  The default behavior is to retrieve as many
+            results as possible per request.
+
+        :type max_results: int
+        :param max_results: The maximum number of results that will
+            be retrieved from Amazon DynamoDB in total.  For example,
+            if you only wanted to see the first 100 results from the
+            query, regardless of how many were actually available, you
+            could set max_results to 100 and the generator returned
+            from the query method will only yeild 100 results max.
+
+        :type count: bool
+        :param count: If True, Amazon DynamoDB returns a total
+            number of items for the Scan operation, even if the
+            operation has no matching items for the assigned filter.
+
+        :type exclusive_start_key: list or tuple
+        :param exclusive_start_key: Primary key of the item from
+            which to continue an earlier query.  This would be
+            provided as the LastEvaluatedKey in that query.
+
+        :type item_class: Class
+        :param item_class: Allows you to override the class used
+            to generate the items. This should be a subclass of
+            :class:`boto.dynamodb.item.Item`
+
+        :return: A TableGenerator (generator) object which will iterate over all results
+        :rtype: :class:`boto.dynamodb.layer2.TableGenerator`
+        """
+        return self.layer2.scan(self, scan_filter, attributes_to_get,
+                                request_limit, max_results, count,
+                                exclusive_start_key, item_class=item_class)
+
+    def batch_get_item(self, keys, attributes_to_get=None):
+        """
+        Return a set of attributes for a multiple items from a single table
+        using their primary keys. This abstraction removes the 100 Items per
+        batch limitations as well as the "UnprocessedKeys" logic.
+
+        :type keys: list
+        :param keys: A list of scalar or tuple values.  Each element in the
+            list represents one Item to retrieve.  If the schema for the
+            table has both a HashKey and a RangeKey, each element in the
+            list should be a tuple consisting of (hash_key, range_key).  If
+            the schema for the table contains only a HashKey, each element
+            in the list should be a scalar value of the appropriate type
+            for the table schema. NOTE: The maximum number of items that
+            can be retrieved for a single operation is 100. Also, the
+            number of items retrieved is constrained by a 1 MB size limit.
+
+        :type attributes_to_get: list
+        :param attributes_to_get: A list of attribute names.
+            If supplied, only the specified attribute names will
+            be returned.  Otherwise, all attributes will be returned.
+
+        :return: A TableBatchGenerator (generator) object which will iterate over all results
+        :rtype: :class:`boto.dynamodb.table.TableBatchGenerator`
+        """
+        return TableBatchGenerator(self, keys, attributes_to_get)