Scripts to download files from google storage based on sha1 sums

third_party/gsutil/boto/tests/unit/glacier/test_layer2.py

Index: third_party/gsutil/boto/tests/unit/glacier/test_layer2.py
diff --git a/third_party/gsutil/boto/tests/unit/glacier/test_layer2.py b/third_party/gsutil/boto/tests/unit/glacier/test_layer2.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a54924a3280ec153467cb3eb3a51ec598a59c90
--- /dev/null
+++ b/third_party/gsutil/boto/tests/unit/glacier/test_layer2.py
@@ -0,0 +1,266 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+from tests.unit import unittest
+
+from mock import call, Mock, patch, sentinel
+
+from boto.glacier.layer1 import Layer1
+from boto.glacier.layer2 import Layer2
+import boto.glacier.vault
+from boto.glacier.vault import Vault
+from boto.glacier.vault import Job
+
+from StringIO import StringIO
+
+# Some fixture data from the Glacier docs
+FIXTURE_VAULT = {
+  "CreationDate" : "2012-02-20T17:01:45.198Z",
+  "LastInventoryDate" : "2012-03-20T17:03:43.221Z",
+  "NumberOfArchives" : 192,
+  "SizeInBytes" : 78088912,
+  "VaultARN" : "arn:aws:glacier:us-east-1:012345678901:vaults/examplevault",
+  "VaultName" : "examplevault"
+}
+
+FIXTURE_ARCHIVE_JOB = {
+  "Action": "ArchiveRetrieval",
+  "ArchiveId": ("NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-TjhqG6eGoOY9Z8i1_AUyUs"
+                "uhPAdTqLHy8pTl5nfCFJmDl2yEZONi5L26Omw12vcs01MNGntHEQL8MBfGlqr"
+                "EXAMPLEArchiveId"),
+  "ArchiveSizeInBytes": 16777216,
+  "Completed": False,
+  "CreationDate": "2012-05-15T17:21:39.339Z",
+  "CompletionDate": "2012-05-15T17:21:43.561Z",
+  "InventorySizeInBytes": None,
+  "JobDescription": "My ArchiveRetrieval Job",
+  "JobId": ("HkF9p6o7yjhFx-K3CGl6fuSm6VzW9T7esGQfco8nUXVYwS0jlb5gq1JZ55yHgt5v"
+            "P54ZShjoQzQVVh7vEXAMPLEjobID"),
+  "SHA256TreeHash": ("beb0fe31a1c7ca8c6c04d574ea906e3f97b31fdca7571defb5b44dc"
+                     "a89b5af60"),
+  "SNSTopic": "arn:aws:sns:us-east-1:012345678901:mytopic",
+  "StatusCode": "InProgress",
+  "StatusMessage": "Operation in progress.",
+  "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/examplevault"
+}
+
+EXAMPLE_PART_LIST_RESULT_PAGE_1 = {
+    "ArchiveDescription": "archive description 1",
+    "CreationDate": "2012-03-20T17:03:43.221Z",
+    "Marker": "MfgsKHVjbQ6EldVl72bn3_n5h2TaGZQUO-Qb3B9j3TITf7WajQ",
+    "MultipartUploadId": "OW2fM5iVylEpFEMM9_HpKowRapC3vn5sSL39_396UW9zLFUWVrnRHaPjUJddQ5OxSHVXjYtrN47NBZ-khxOjyEXAMPLE",
+    "PartSizeInBytes": 4194304,
+    "Parts":
+    [ {
+      "RangeInBytes": "4194304-8388607",
+      "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4"
+      }],
+    "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/demo1-vault"
+}
+
+# The documentation doesn't say whether the non-Parts fields are defined in
+# future pages, so assume they are not.
+EXAMPLE_PART_LIST_RESULT_PAGE_2 = {
+    "ArchiveDescription": None,
+    "CreationDate": None,
+    "Marker": None,
+    "MultipartUploadId": None,
+    "PartSizeInBytes": None,
+    "Parts":
+    [ {
+      "RangeInBytes": "0-4194303",
+      "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4"
+      }],
+    "VaultARN": None
+}
+
+EXAMPLE_PART_LIST_COMPLETE = {
+    "ArchiveDescription": "archive description 1",
+    "CreationDate": "2012-03-20T17:03:43.221Z",
+    "Marker": None,
+    "MultipartUploadId": "OW2fM5iVylEpFEMM9_HpKowRapC3vn5sSL39_396UW9zLFUWVrnRHaPjUJddQ5OxSHVXjYtrN47NBZ-khxOjyEXAMPLE",
+    "PartSizeInBytes": 4194304,
+    "Parts":
+    [ {
+      "RangeInBytes": "4194304-8388607",
+      "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4"
+    }, {
+      "RangeInBytes": "0-4194303",
+      "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4"
+      }],
+    "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/demo1-vault"
+}
+
+
+class GlacierLayer2Base(unittest.TestCase):
+    def setUp(self):
+        self.mock_layer1 = Mock(spec=Layer1)
+
+
+class TestGlacierLayer2Connection(GlacierLayer2Base):
+    def setUp(self):
+        GlacierLayer2Base.setUp(self)
+        self.layer2 = Layer2(layer1=self.mock_layer1)
+
+    def test_create_vault(self):
+        self.mock_layer1.describe_vault.return_value = FIXTURE_VAULT
+        self.layer2.create_vault("My Vault")
+        self.mock_layer1.create_vault.assert_called_with("My Vault")
+
+    def test_get_vault(self):
+        self.mock_layer1.describe_vault.return_value = FIXTURE_VAULT
+        vault = self.layer2.get_vault("examplevault")
+        self.assertEqual(vault.layer1, self.mock_layer1)
+        self.assertEqual(vault.name, "examplevault")
+        self.assertEqual(vault.size, 78088912)
+        self.assertEqual(vault.number_of_archives, 192)
+
+    def list_vaults(self):
+        self.mock_layer1.list_vaults.return_value = [FIXTURE_VAULT]
+        vaults = self.layer2.list_vaults()
+        self.assertEqual(vaults[0].name, "examplevault")
+
+
+class TestVault(GlacierLayer2Base):
+    def setUp(self):
+        GlacierLayer2Base.setUp(self)
+        self.vault = Vault(self.mock_layer1, FIXTURE_VAULT)
+
+    # TODO: Tests for the other methods of uploading
+
+    def test_create_archive_writer(self):
+        self.mock_layer1.initiate_multipart_upload.return_value = {
+            "UploadId": "UPLOADID"}
+        writer = self.vault.create_archive_writer(description="stuff")
+        self.mock_layer1.initiate_multipart_upload.assert_called_with(
+            "examplevault", self.vault.DefaultPartSize, "stuff")
+        self.assertEqual(writer.vault, self.vault)
+        self.assertEqual(writer.upload_id, "UPLOADID")
+
+    def test_delete_vault(self):
+        self.vault.delete_archive("archive")
+        self.mock_layer1.delete_archive.assert_called_with("examplevault",
+                                                           "archive")
+
+    def test_get_job(self):
+        self.mock_layer1.describe_job.return_value = FIXTURE_ARCHIVE_JOB
+        job = self.vault.get_job(
+            "NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-TjhqG6eGoOY9Z8i1_AUyUsuhPA"
+            "dTqLHy8pTl5nfCFJmDl2yEZONi5L26Omw12vcs01MNGntHEQL8MBfGlqrEXAMPLEA"
+            "rchiveId")
+        self.assertEqual(job.action, "ArchiveRetrieval")
+
+    def test_list_jobs(self):
+        self.mock_layer1.list_jobs.return_value = {
+            "JobList": [FIXTURE_ARCHIVE_JOB]}
+        jobs = self.vault.list_jobs(False, "InProgress")
+        self.mock_layer1.list_jobs.assert_called_with("examplevault",
+                                                      False, "InProgress")
+        self.assertEqual(jobs[0].archive_id,
+                         "NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-TjhqG6eGoOY9Z"
+                         "8i1_AUyUsuhPAdTqLHy8pTl5nfCFJmDl2yEZONi5L26Omw12vcs0"
+                         "1MNGntHEQL8MBfGlqrEXAMPLEArchiveId")
+
+    def test_list_all_parts_one_page(self):
+        self.mock_layer1.list_parts.return_value = (
+            dict(EXAMPLE_PART_LIST_COMPLETE)) # take a copy
+        parts_result = self.vault.list_all_parts(sentinel.upload_id)
+        expected = [call('examplevault', sentinel.upload_id)]
+        self.assertEquals(expected, self.mock_layer1.list_parts.call_args_list)
+        self.assertEquals(EXAMPLE_PART_LIST_COMPLETE, parts_result)
+
+    def test_list_all_parts_two_pages(self):
+        self.mock_layer1.list_parts.side_effect = [
+            # take copies
+            dict(EXAMPLE_PART_LIST_RESULT_PAGE_1),
+            dict(EXAMPLE_PART_LIST_RESULT_PAGE_2)
+        ]
+        parts_result = self.vault.list_all_parts(sentinel.upload_id)
+        expected = [call('examplevault', sentinel.upload_id),
+                    call('examplevault', sentinel.upload_id,
+                         marker=EXAMPLE_PART_LIST_RESULT_PAGE_1['Marker'])]
+        self.assertEquals(expected, self.mock_layer1.list_parts.call_args_list)
+        self.assertEquals(EXAMPLE_PART_LIST_COMPLETE, parts_result)
+
+    @patch('boto.glacier.vault.resume_file_upload')
+    def test_resume_archive_from_file(self, mock_resume_file_upload):
+        part_size = 4
+        mock_list_parts = Mock()
+        mock_list_parts.return_value = {
+            'PartSizeInBytes': part_size,
+            'Parts': [{
+                'RangeInBytes': '0-3',
+                'SHA256TreeHash': '12',
+                }, {
+                'RangeInBytes': '4-6',
+                'SHA256TreeHash': '34',
+                },
+        ]}
+
+        self.vault.list_all_parts = mock_list_parts
+        self.vault.resume_archive_from_file(
+            sentinel.upload_id, file_obj=sentinel.file_obj)
+        mock_resume_file_upload.assert_called_once_with(
+            self.vault, sentinel.upload_id, part_size, sentinel.file_obj,
+            {0: '12'.decode('hex'), 1: '34'.decode('hex')})
+
+
+class TestJob(GlacierLayer2Base):
+    def setUp(self):
+        GlacierLayer2Base.setUp(self)
+        self.vault = Vault(self.mock_layer1, FIXTURE_VAULT)
+        self.job = Job(self.vault, FIXTURE_ARCHIVE_JOB)
+
+    def test_get_job_output(self):
+        self.mock_layer1.get_job_output.return_value = "TEST_OUTPUT"
+        self.job.get_output((0,100))
+        self.mock_layer1.get_job_output.assert_called_with(
+            "examplevault",
+            "HkF9p6o7yjhFx-K3CGl6fuSm6VzW9T7esGQfco8nUXVYwS0jlb5gq1JZ55yHgt5vP"
+            "54ZShjoQzQVVh7vEXAMPLEjobID", (0,100))
+
+class TestRangeStringParsing(unittest.TestCase):
+    def test_simple_range(self):
+        self.assertEquals(
+            Vault._range_string_to_part_index('0-3', 4), 0)
+
+    def test_range_one_too_big(self):
+        # Off-by-one bug in Amazon's Glacier implementation
+        # See: https://forums.aws.amazon.com/thread.jspa?threadID=106866&tstart=0
+        # Workaround is to assume that if a (start, end] range appears to be
+        # returned then that is what it is.
+        self.assertEquals(
+            Vault._range_string_to_part_index('0-4', 4), 0)
+
+    def test_range_too_big(self):
+        self.assertRaises(
+            AssertionError, Vault._range_string_to_part_index, '0-5', 4)
+
+    def test_range_start_mismatch(self):
+        self.assertRaises(
+            AssertionError, Vault._range_string_to_part_index, '1-3', 4)
+
+    def test_range_end_mismatch(self):
+        # End mismatch is OK, since the last part might be short
+        self.assertEquals(
+            Vault._range_string_to_part_index('0-2', 4), 0)