Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(214)

Side by Side Diff: third_party/gsutil/boto/tests/s3/test_resumable_uploads.py

Issue 10199002: Upgrade gsutil to 3.4 (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Addressed comments Created 8 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # 2 #
3 # Copyright 2010 Google Inc. 3 # Copyright 2010 Google Inc.
4 # 4 #
5 # Permission is hereby granted, free of charge, to any person obtaining a 5 # Permission is hereby granted, free of charge, to any person obtaining a
6 # copy of this software and associated documentation files (the 6 # copy of this software and associated documentation files (the
7 # "Software"), to deal in the Software without restriction, including 7 # "Software"), to deal in the Software without restriction, including
8 # without limitation the rights to use, copy, modify, merge, publish, dis- 8 # without limitation the rights to use, copy, modify, merge, publish, dis-
9 # tribute, sublicense, and/or sell copies of the Software, and to permit 9 # tribute, sublicense, and/or sell copies of the Software, and to permit
10 # persons to whom the Software is furnished to do so, subject to the fol- 10 # persons to whom the Software is furnished to do so, subject to the fol-
11 # lowing conditions: 11 # lowing conditions:
12 # 12 #
13 # The above copyright notice and this permission notice shall be included 13 # The above copyright notice and this permission notice shall be included
14 # in all copies or substantial portions of the Software. 14 # in all copies or substantial portions of the Software.
15 # 15 #
16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 17 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
18 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 18 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
19 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 # IN THE SOFTWARE. 22 # IN THE SOFTWARE.
23 23
24 """ 24 """
25 Tests of Google Storage resumable uploads. 25 Tests of Google Cloud Storage resumable uploads.
26 """ 26 """
27 27
28 import errno 28 import errno
29 import getopt 29 import getopt
30 import os 30 import os
31 import random 31 import random
32 import re 32 import re
33 import shutil 33 import shutil
34 import socket 34 import socket
35 import StringIO 35 import StringIO
36 import sys 36 import sys
37 import tempfile 37 import tempfile
38 import time 38 import time
39 import unittest 39 import unittest
40 40
41 import boto 41 import boto
42 from boto.exception import GSResponseError 42 from boto.exception import GSResponseError
43 from boto.gs.resumable_upload_handler import ResumableUploadHandler 43 from boto.gs.resumable_upload_handler import ResumableUploadHandler
44 from boto.exception import InvalidUriError
44 from boto.exception import ResumableTransferDisposition 45 from boto.exception import ResumableTransferDisposition
45 from boto.exception import ResumableUploadException 46 from boto.exception import ResumableUploadException
46 from boto.exception import StorageResponseError 47 from boto.exception import StorageResponseError
47 from tests.s3.cb_test_harnass import CallbackTestHarnass 48 from cb_test_harnass import CallbackTestHarnass
48 49
49 # We don't use the OAuth2 authentication plugin directly; importing it here 50 # We don't use the OAuth2 authentication plugin directly; importing it here
50 # ensures that it's loaded and available by default. 51 # ensures that it's loaded and available by default.
51 try: 52 try:
52 from oauth2_plugin import oauth2_plugin 53 from oauth2_plugin import oauth2_plugin
53 except ImportError: 54 except ImportError:
54 # Do nothing - if user doesn't have OAuth2 configured it doesn't matter; 55 # Do nothing - if user doesn't have OAuth2 configured it doesn't matter;
55 # and if they do, the tests will fail (as they should in that case). 56 # and if they do, the tests will fail (as they should in that case).
56 pass 57 pass
57 58
58 59
59 class ResumableUploadTests(unittest.TestCase): 60 class ResumableUploadTests(unittest.TestCase):
60 """ 61 """
61 Resumable upload test suite. 62 Resumable upload test suite.
62 """ 63 """
63 64
64 def get_suite_description(self): 65 def get_suite_description(self):
65 return 'Resumable upload test suite' 66 return 'Resumable upload test suite'
66 67
67 @classmethod 68 def build_test_input_file(self, size):
68 def setUp(cls):
69 """
70 Creates dst_key needed by all tests.
71
72 This method's namingCase is required by the unittest framework.
73 """
74 cls.dst_key = cls.dst_key_uri.new_key(validate=False)
75
76 @classmethod
77 def tearDown(cls):
78 """
79 Deletes any objects or files created by last test run.
80
81 This method's namingCase is required by the unittest framework.
82 """
83 try:
84 cls.dst_key_uri.delete_key()
85 except GSResponseError:
86 # Ignore possible not-found error.
87 pass
88 # Recursively delete dst dir and then re-create it, so in effect we
89 # remove all dirs and files under that directory.
90 shutil.rmtree(cls.tmp_dir)
91 os.mkdir(cls.tmp_dir)
92
93 @staticmethod
94 def build_test_input_file(size):
95 buf = [] 69 buf = []
96 # I manually construct the random data here instead of calling 70 # I manually construct the random data here instead of calling
97 # os.urandom() because I want to constrain the range of data (in 71 # os.urandom() because I want to constrain the range of data (in
98 # this case to 0'..'9') so the test 72 # this case to 0'..'9') so the test
99 # code can easily overwrite part of the StringIO file with 73 # code can easily overwrite part of the StringIO file with
100 # known-to-be-different values. 74 # known-to-be-different values.
101 for i in range(size): 75 for i in range(size):
102 buf.append(str(random.randint(0, 9))) 76 buf.append(str(random.randint(0, 9)))
103 file_as_string = ''.join(buf) 77 file_as_string = ''.join(buf)
104 return (file_as_string, StringIO.StringIO(file_as_string)) 78 return (file_as_string, StringIO.StringIO(file_as_string))
105 79
106 @classmethod 80 def get_dst_bucket_uri(self):
107 def set_up_class(cls, debug): 81 """A unique bucket to test."""
82 hostname = socket.gethostname().split('.')[0]
83 uri_base_str = 'gs://res-upload-test-%s-%s-%s' % (
84 hostname, os.getpid(), int(time.time()))
85 return boto.storage_uri('%s-dst' % uri_base_str)
86
87 def get_dst_key_uri(self):
88 """A key to test."""
89 return self.dst_bucket_uri.clone_replace_name('obj')
90
91 def get_staged_host(self):
92 """URL of an existing bucket."""
93 return 'pub.commondatastorage.googleapis.com'
94
95 def get_invalid_upload_id(self):
96 return (
97 'http://%s/?upload_id='
98 'AyzB2Uo74W4EYxyi5dp_-r68jz8rtbvshsv4TX7srJVkJ57CxTY5Dw2' % (
99 self.get_staged_host()))
100
101 def setUp(self):
108 """ 102 """
109 Initializes test suite. 103 Creates dst bucket and data needed by each test.
110 """ 104 """
111
112 # Use a designated tmpdir prefix to make it easy to find the end of 105 # Use a designated tmpdir prefix to make it easy to find the end of
113 # the tmp path. 106 # the tmp path.
114 cls.tmpdir_prefix = 'tmp_resumable_upload_test' 107 self.tmpdir_prefix = 'tmp_resumable_upload_test'
115 108
116 # Create test source file data. 109 # Create test source file data.
117 cls.empty_src_file_size = 0 110 self.empty_src_file_size = 0
118 (cls.empty_src_file_as_string, cls.empty_src_file) = ( 111 (self.empty_src_file_as_string, self.empty_src_file) = (
119 cls.build_test_input_file(cls.empty_src_file_size)) 112 self.build_test_input_file(self.empty_src_file_size))
120 cls.small_src_file_size = 2 * 1024 # 2 KB. 113 self.small_src_file_size = 2 * 1024 # 2 KB.
121 (cls.small_src_file_as_string, cls.small_src_file) = ( 114 (self.small_src_file_as_string, self.small_src_file) = (
122 cls.build_test_input_file(cls.small_src_file_size)) 115 self.build_test_input_file(self.small_src_file_size))
123 cls.larger_src_file_size = 500 * 1024 # 500 KB. 116 self.larger_src_file_size = 500 * 1024 # 500 KB.
124 (cls.larger_src_file_as_string, cls.larger_src_file) = ( 117 (self.larger_src_file_as_string, self.larger_src_file) = (
125 cls.build_test_input_file(cls.larger_src_file_size)) 118 self.build_test_input_file(self.larger_src_file_size))
126 cls.largest_src_file_size = 1024 * 1024 # 1 MB. 119 self.largest_src_file_size = 1024 * 1024 # 1 MB.
127 (cls.largest_src_file_as_string, cls.largest_src_file) = ( 120 (self.largest_src_file_as_string, self.largest_src_file) = (
128 cls.build_test_input_file(cls.largest_src_file_size)) 121 self.build_test_input_file(self.largest_src_file_size))
129 122
130 # Create temp dir. 123 # Create temp dir.
131 cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix) 124 self.tmp_dir = tempfile.mkdtemp(prefix=self.tmpdir_prefix)
132 125
133 # Create the test bucket. 126 # Create the test bucket.
134 hostname = socket.gethostname().split('.')[0] 127 self.dst_bucket_uri = self.get_dst_bucket_uri()
135 cls.uri_base_str = 'gs://res_upload_test_%s_%s_%s' % ( 128 self.dst_bucket_uri.create_bucket()
136 hostname, os.getpid(), int(time.time())) 129 self.dst_key_uri = self.get_dst_key_uri()
137 cls.dst_bucket_uri = boto.storage_uri('%s_dst' %
138 cls.uri_base_str, debug=debug)
139 cls.dst_bucket_uri.create_bucket()
140 cls.dst_key_uri = cls.dst_bucket_uri.clone_replace_name('obj')
141 130
142 cls.tracker_file_name = '%s%suri_tracker' % (cls.tmp_dir, os.sep) 131 self.tracker_file_name = '%s%suri_tracker' % (self.tmp_dir, os.sep)
143 132
144 cls.syntactically_invalid_tracker_file_name = ( 133 self.syntactically_invalid_tracker_file_name = (
145 '%s%ssynt_invalid_uri_tracker' % (cls.tmp_dir, os.sep)) 134 '%s%ssynt_invalid_uri_tracker' % (self.tmp_dir, os.sep))
146 f = open(cls.syntactically_invalid_tracker_file_name, 'w') 135 f = open(self.syntactically_invalid_tracker_file_name, 'w')
147 f.write('ftp://example.com') 136 f.write('ftp://example.com')
148 f.close() 137 f.close()
149 138
150 cls.invalid_upload_id = ( 139 self.invalid_upload_id = self.get_invalid_upload_id()
151 'http://pub.commondatastorage.googleapis.com/?upload_id=' 140 self.invalid_upload_id_tracker_file_name = (
152 'AyzB2Uo74W4EYxyi5dp_-r68jz8rtbvshsv4TX7srJVkJ57CxTY5Dw2') 141 '%s%sinvalid_upload_id_tracker' % (self.tmp_dir, os.sep))
153 cls.invalid_upload_id_tracker_file_name = ( 142 f = open(self.invalid_upload_id_tracker_file_name, 'w')
154 '%s%sinvalid_upload_id_tracker' % (cls.tmp_dir, os.sep)) 143 f.write(self.invalid_upload_id)
155 f = open(cls.invalid_upload_id_tracker_file_name, 'w')
156 f.write(cls.invalid_upload_id)
157 f.close() 144 f.close()
158 145
159 cls.created_test_data = True 146 self.dst_key = self.dst_key_uri.new_key(validate=False)
147 self.created_test_data = True
160 148
161 @classmethod 149 def tearDown(self):
162 def tear_down_class(cls):
163 """ 150 """
164 Deletes bucket and tmp dir created by set_up_class. 151 Deletes any objects, files, and bucket from each test run.
165 """ 152 """
166 if not hasattr(cls, 'created_test_data'): 153 if not hasattr(self, 'created_test_data'):
167 return 154 return
168 # Call cls.tearDown() in case the tests got interrupted, to ensure 155
169 # dst objects get deleted. 156 shutil.rmtree(self.tmp_dir)
170 cls.tearDown()
171 157
172 # Retry (for up to 2 minutes) the bucket gets deleted (it may not 158 # Retry (for up to 2 minutes) the bucket gets deleted (it may not
173 # the first time round, due to eventual consistency of bucket delete 159 # the first time round, due to eventual consistency of bucket delete
174 # operations). 160 # operations). We also retry key deletions because if the key fails
161 # to be deleted on the first attempt, it will stop us from deleting
162 # the bucket.
175 for i in range(60): 163 for i in range(60):
176 try: 164 try:
177 cls.dst_bucket_uri.delete_bucket() 165 self.dst_key_uri.delete_key()
166 except GSResponseError, e:
167 # Ignore errors attempting to delete the key, because not all
168 # tests will write to the dst key.
169 pass
170 try:
171 self.dst_bucket_uri.delete_bucket()
178 break 172 break
179 except StorageResponseError: 173 except StorageResponseError:
180 print 'Test bucket (%s) not yet deleted, still trying' % ( 174 print 'Test bucket (%s) not yet deleted, still trying' % (
181 cls.dst_bucket_uri.uri) 175 self.dst_bucket_uri.uri)
182 time.sleep(2) 176 time.sleep(2)
183 shutil.rmtree(cls.tmp_dir) 177 shutil.rmtree(self.tmp_dir, ignore_errors=True)
184 cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix) 178 self.tmp_dir = tempfile.mkdtemp(prefix=self.tmpdir_prefix)
185 179
186 def test_non_resumable_upload(self): 180 def test_non_resumable_upload(self):
187 """ 181 """
188 Tests that non-resumable uploads work 182 Tests that non-resumable uploads work
189 """ 183 """
190 self.dst_key.set_contents_from_file(self.small_src_file) 184 # Seek to end incase its the first test.
185 self.small_src_file.seek(0, os.SEEK_END)
186 try:
187 self.dst_key.set_contents_from_file(self.small_src_file)
188 self.fail("should fail as need to rewind the filepointer")
189 except AttributeError:
190 pass
191 # Now try calling with a proper rewind.
192 self.dst_key.set_contents_from_file(self.small_src_file, rewind=True)
191 self.assertEqual(self.small_src_file_size, self.dst_key.size) 193 self.assertEqual(self.small_src_file_size, self.dst_key.size)
192 self.assertEqual(self.small_src_file_as_string, 194 self.assertEqual(self.small_src_file_as_string,
193 self.dst_key.get_contents_as_string()) 195 self.dst_key.get_contents_as_string())
194 196
195 def test_upload_without_persistent_tracker(self): 197 def test_upload_without_persistent_tracker(self):
196 """ 198 """
197 Tests a single resumable upload, with no tracker URI persistence 199 Tests a single resumable upload, with no tracker URI persistence
198 """ 200 """
199 res_upload_handler = ResumableUploadHandler() 201 res_upload_handler = ResumableUploadHandler()
202 self.small_src_file.seek(0)
200 self.dst_key.set_contents_from_file( 203 self.dst_key.set_contents_from_file(
201 self.small_src_file, res_upload_handler=res_upload_handler) 204 self.small_src_file, res_upload_handler=res_upload_handler)
202 self.assertEqual(self.small_src_file_size, self.dst_key.size) 205 self.assertEqual(self.small_src_file_size, self.dst_key.size)
203 self.assertEqual(self.small_src_file_as_string, 206 self.assertEqual(self.small_src_file_as_string,
204 self.dst_key.get_contents_as_string()) 207 self.dst_key.get_contents_as_string())
205 208
206 def test_failed_upload_with_persistent_tracker(self): 209 def test_failed_upload_with_persistent_tracker(self):
207 """ 210 """
208 Tests that failed resumable upload leaves a correct tracker URI file 211 Tests that failed resumable upload leaves a correct tracker URI file
209 """ 212 """
210 harnass = CallbackTestHarnass() 213 harnass = CallbackTestHarnass()
211 res_upload_handler = ResumableUploadHandler( 214 res_upload_handler = ResumableUploadHandler(
212 tracker_file_name=self.tracker_file_name, num_retries=0) 215 tracker_file_name=self.tracker_file_name, num_retries=0)
216 self.small_src_file.seek(0)
213 try: 217 try:
214 self.dst_key.set_contents_from_file( 218 self.dst_key.set_contents_from_file(
215 self.small_src_file, cb=harnass.call, 219 self.small_src_file, cb=harnass.call,
216 res_upload_handler=res_upload_handler) 220 res_upload_handler=res_upload_handler)
217 self.fail('Did not get expected ResumableUploadException') 221 self.fail('Did not get expected ResumableUploadException')
218 except ResumableUploadException, e: 222 except ResumableUploadException, e:
219 # We'll get a ResumableUploadException at this point because 223 # We'll get a ResumableUploadException at this point because
220 # of CallbackTestHarnass (above). Check that the tracker file was 224 # of CallbackTestHarnass (above). Check that the tracker file was
221 # created correctly. 225 # created correctly.
222 self.assertEqual(e.disposition, 226 self.assertEqual(e.disposition,
223 ResumableTransferDisposition.ABORT_CUR_PROCESS) 227 ResumableTransferDisposition.ABORT_CUR_PROCESS)
224 self.assertTrue(os.path.exists(self.tracker_file_name)) 228 self.assertTrue(os.path.exists(self.tracker_file_name))
225 f = open(self.tracker_file_name) 229 f = open(self.tracker_file_name)
226 uri_from_file = f.readline().strip() 230 uri_from_file = f.readline().strip()
227 f.close() 231 f.close()
228 self.assertEqual(uri_from_file, 232 self.assertEqual(uri_from_file,
229 res_upload_handler.get_tracker_uri()) 233 res_upload_handler.get_tracker_uri())
230 234
231 def test_retryable_exception_recovery(self): 235 def test_retryable_exception_recovery(self):
232 """ 236 """
233 Tests handling of a retryable exception 237 Tests handling of a retryable exception
234 """ 238 """
235 # Test one of the RETRYABLE_EXCEPTIONS. 239 # Test one of the RETRYABLE_EXCEPTIONS.
236 exception = ResumableUploadHandler.RETRYABLE_EXCEPTIONS[0] 240 exception = ResumableUploadHandler.RETRYABLE_EXCEPTIONS[0]
237 harnass = CallbackTestHarnass(exception=exception) 241 harnass = CallbackTestHarnass(exception=exception)
238 res_upload_handler = ResumableUploadHandler(num_retries=1) 242 res_upload_handler = ResumableUploadHandler(num_retries=1)
243 self.small_src_file.seek(0)
239 self.dst_key.set_contents_from_file( 244 self.dst_key.set_contents_from_file(
240 self.small_src_file, cb=harnass.call, 245 self.small_src_file, cb=harnass.call,
241 res_upload_handler=res_upload_handler) 246 res_upload_handler=res_upload_handler)
242 # Ensure uploaded object has correct content. 247 # Ensure uploaded object has correct content.
243 self.assertEqual(self.small_src_file_size, self.dst_key.size) 248 self.assertEqual(self.small_src_file_size, self.dst_key.size)
244 self.assertEqual(self.small_src_file_as_string, 249 self.assertEqual(self.small_src_file_as_string,
245 self.dst_key.get_contents_as_string()) 250 self.dst_key.get_contents_as_string())
246 251
247 def test_broken_pipe_recovery(self): 252 def test_broken_pipe_recovery(self):
248 """ 253 """
249 Tests handling of a Broken Pipe (which interacts with an httplib bug) 254 Tests handling of a Broken Pipe (which interacts with an httplib bug)
250 """ 255 """
251 exception = IOError(errno.EPIPE, "Broken pipe") 256 exception = IOError(errno.EPIPE, "Broken pipe")
252 harnass = CallbackTestHarnass(exception=exception) 257 harnass = CallbackTestHarnass(exception=exception)
253 res_upload_handler = ResumableUploadHandler(num_retries=1) 258 res_upload_handler = ResumableUploadHandler(num_retries=1)
259 self.small_src_file.seek(0)
254 self.dst_key.set_contents_from_file( 260 self.dst_key.set_contents_from_file(
255 self.small_src_file, cb=harnass.call, 261 self.small_src_file, cb=harnass.call,
256 res_upload_handler=res_upload_handler) 262 res_upload_handler=res_upload_handler)
257 # Ensure uploaded object has correct content. 263 # Ensure uploaded object has correct content.
258 self.assertEqual(self.small_src_file_size, self.dst_key.size) 264 self.assertEqual(self.small_src_file_size, self.dst_key.size)
259 self.assertEqual(self.small_src_file_as_string, 265 self.assertEqual(self.small_src_file_as_string,
260 self.dst_key.get_contents_as_string()) 266 self.dst_key.get_contents_as_string())
261 267
262 def test_non_retryable_exception_handling(self): 268 def test_non_retryable_exception_handling(self):
263 """ 269 """
264 Tests a resumable upload that fails with a non-retryable exception 270 Tests a resumable upload that fails with a non-retryable exception
265 """ 271 """
266 harnass = CallbackTestHarnass( 272 harnass = CallbackTestHarnass(
267 exception=OSError(errno.EACCES, 'Permission denied')) 273 exception=OSError(errno.EACCES, 'Permission denied'))
268 res_upload_handler = ResumableUploadHandler(num_retries=1) 274 res_upload_handler = ResumableUploadHandler(num_retries=1)
275 self.small_src_file.seek(0)
269 try: 276 try:
270 self.dst_key.set_contents_from_file( 277 self.dst_key.set_contents_from_file(
271 self.small_src_file, cb=harnass.call, 278 self.small_src_file, cb=harnass.call,
272 res_upload_handler=res_upload_handler) 279 res_upload_handler=res_upload_handler)
273 self.fail('Did not get expected OSError') 280 self.fail('Did not get expected OSError')
274 except OSError, e: 281 except OSError, e:
275 # Ensure the error was re-raised. 282 # Ensure the error was re-raised.
276 self.assertEqual(e.errno, 13) 283 self.assertEqual(e.errno, 13)
277 284
278 def test_failed_and_restarted_upload_with_persistent_tracker(self): 285 def test_failed_and_restarted_upload_with_persistent_tracker(self):
279 """ 286 """
280 Tests resumable upload that fails once and then completes, with tracker 287 Tests resumable upload that fails once and then completes, with tracker
281 file 288 file
282 """ 289 """
283 harnass = CallbackTestHarnass() 290 harnass = CallbackTestHarnass()
284 res_upload_handler = ResumableUploadHandler( 291 res_upload_handler = ResumableUploadHandler(
285 tracker_file_name=self.tracker_file_name, num_retries=1) 292 tracker_file_name=self.tracker_file_name, num_retries=1)
293 self.small_src_file.seek(0)
286 self.dst_key.set_contents_from_file( 294 self.dst_key.set_contents_from_file(
287 self.small_src_file, cb=harnass.call, 295 self.small_src_file, cb=harnass.call,
288 res_upload_handler=res_upload_handler) 296 res_upload_handler=res_upload_handler)
289 # Ensure uploaded object has correct content. 297 # Ensure uploaded object has correct content.
290 self.assertEqual(self.small_src_file_size, self.dst_key.size) 298 self.assertEqual(self.small_src_file_size, self.dst_key.size)
291 self.assertEqual(self.small_src_file_as_string, 299 self.assertEqual(self.small_src_file_as_string,
292 self.dst_key.get_contents_as_string()) 300 self.dst_key.get_contents_as_string())
293 # Ensure tracker file deleted. 301 # Ensure tracker file deleted.
294 self.assertFalse(os.path.exists(self.tracker_file_name)) 302 self.assertFalse(os.path.exists(self.tracker_file_name))
295 303
296 def test_multiple_in_process_failures_then_succeed(self): 304 def test_multiple_in_process_failures_then_succeed(self):
297 """ 305 """
298 Tests resumable upload that fails twice in one process, then completes 306 Tests resumable upload that fails twice in one process, then completes
299 """ 307 """
300 res_upload_handler = ResumableUploadHandler(num_retries=3) 308 res_upload_handler = ResumableUploadHandler(num_retries=3)
309 self.small_src_file.seek(0)
301 self.dst_key.set_contents_from_file( 310 self.dst_key.set_contents_from_file(
302 self.small_src_file, res_upload_handler=res_upload_handler) 311 self.small_src_file, res_upload_handler=res_upload_handler)
303 # Ensure uploaded object has correct content. 312 # Ensure uploaded object has correct content.
304 self.assertEqual(self.small_src_file_size, self.dst_key.size) 313 self.assertEqual(self.small_src_file_size, self.dst_key.size)
305 self.assertEqual(self.small_src_file_as_string, 314 self.assertEqual(self.small_src_file_as_string,
306 self.dst_key.get_contents_as_string()) 315 self.dst_key.get_contents_as_string())
307 316
308 def test_multiple_in_process_failures_then_succeed_with_tracker_file(self): 317 def test_multiple_in_process_failures_then_succeed_with_tracker_file(self):
309 """ 318 """
310 Tests resumable upload that fails completely in one process, 319 Tests resumable upload that fails completely in one process,
311 then when restarted completes, using a tracker file 320 then when restarted completes, using a tracker file
312 """ 321 """
313 # Set up test harnass that causes more failures than a single 322 # Set up test harnass that causes more failures than a single
314 # ResumableUploadHandler instance will handle, writing enough data 323 # ResumableUploadHandler instance will handle, writing enough data
315 # before the first failure that some of it survives that process run. 324 # before the first failure that some of it survives that process run.
316 harnass = CallbackTestHarnass( 325 harnass = CallbackTestHarnass(
317 fail_after_n_bytes=self.larger_src_file_size/2, num_times_to_fail=2) 326 fail_after_n_bytes=self.larger_src_file_size/2, num_times_to_fail=2)
318 res_upload_handler = ResumableUploadHandler( 327 res_upload_handler = ResumableUploadHandler(
319 tracker_file_name=self.tracker_file_name, num_retries=1) 328 tracker_file_name=self.tracker_file_name, num_retries=1)
329 self.larger_src_file.seek(0)
320 try: 330 try:
321 self.dst_key.set_contents_from_file( 331 self.dst_key.set_contents_from_file(
322 self.larger_src_file, cb=harnass.call, 332 self.larger_src_file, cb=harnass.call,
323 res_upload_handler=res_upload_handler) 333 res_upload_handler=res_upload_handler)
324 self.fail('Did not get expected ResumableUploadException') 334 self.fail('Did not get expected ResumableUploadException')
325 except ResumableUploadException, e: 335 except ResumableUploadException, e:
326 self.assertEqual(e.disposition, 336 self.assertEqual(e.disposition,
327 ResumableTransferDisposition.ABORT_CUR_PROCESS) 337 ResumableTransferDisposition.ABORT_CUR_PROCESS)
328 # Ensure a tracker file survived. 338 # Ensure a tracker file survived.
329 self.assertTrue(os.path.exists(self.tracker_file_name)) 339 self.assertTrue(os.path.exists(self.tracker_file_name))
330 # Try it one more time; this time should succeed. 340 # Try it one more time; this time should succeed.
341 self.larger_src_file.seek(0)
331 self.dst_key.set_contents_from_file( 342 self.dst_key.set_contents_from_file(
332 self.larger_src_file, cb=harnass.call, 343 self.larger_src_file, cb=harnass.call,
333 res_upload_handler=res_upload_handler) 344 res_upload_handler=res_upload_handler)
334 self.assertEqual(self.larger_src_file_size, self.dst_key.size) 345 self.assertEqual(self.larger_src_file_size, self.dst_key.size)
335 self.assertEqual(self.larger_src_file_as_string, 346 self.assertEqual(self.larger_src_file_as_string,
336 self.dst_key.get_contents_as_string()) 347 self.dst_key.get_contents_as_string())
337 self.assertFalse(os.path.exists(self.tracker_file_name)) 348 self.assertFalse(os.path.exists(self.tracker_file_name))
338 # Ensure some of the file was uploaded both before and after failure. 349 # Ensure some of the file was uploaded both before and after failure.
339 self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1 350 self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1
340 and 351 and
341 len(harnass.transferred_seq_after_first_failure) > 1) 352 len(harnass.transferred_seq_after_first_failure) > 1)
342 353
343 def test_upload_with_inital_partial_upload_before_failure(self): 354 def test_upload_with_inital_partial_upload_before_failure(self):
344 """ 355 """
345 Tests resumable upload that successfully uploads some content 356 Tests resumable upload that successfully uploads some content
346 before it fails, then restarts and completes 357 before it fails, then restarts and completes
347 """ 358 """
348 # Set up harnass to fail upload after several hundred KB so upload 359 # Set up harnass to fail upload after several hundred KB so upload
349 # server will have saved something before we retry. 360 # server will have saved something before we retry.
350 harnass = CallbackTestHarnass( 361 harnass = CallbackTestHarnass(
351 fail_after_n_bytes=self.larger_src_file_size/2) 362 fail_after_n_bytes=self.larger_src_file_size/2)
352 res_upload_handler = ResumableUploadHandler(num_retries=1) 363 res_upload_handler = ResumableUploadHandler(num_retries=1)
364 self.larger_src_file.seek(0)
353 self.dst_key.set_contents_from_file( 365 self.dst_key.set_contents_from_file(
354 self.larger_src_file, cb=harnass.call, 366 self.larger_src_file, cb=harnass.call,
355 res_upload_handler=res_upload_handler) 367 res_upload_handler=res_upload_handler)
356 # Ensure uploaded object has correct content. 368 # Ensure uploaded object has correct content.
357 self.assertEqual(self.larger_src_file_size, self.dst_key.size) 369 self.assertEqual(self.larger_src_file_size, self.dst_key.size)
358 self.assertEqual(self.larger_src_file_as_string, 370 self.assertEqual(self.larger_src_file_as_string,
359 self.dst_key.get_contents_as_string()) 371 self.dst_key.get_contents_as_string())
360 # Ensure some of the file was uploaded both before and after failure. 372 # Ensure some of the file was uploaded both before and after failure.
361 self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1 373 self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1
362 and 374 and
363 len(harnass.transferred_seq_after_first_failure) > 1) 375 len(harnass.transferred_seq_after_first_failure) > 1)
364 376
365 def test_empty_file_upload(self): 377 def test_empty_file_upload(self):
366 """ 378 """
367 Tests uploading an empty file (exercises boundary conditions). 379 Tests uploading an empty file (exercises boundary conditions).
368 """ 380 """
369 res_upload_handler = ResumableUploadHandler() 381 res_upload_handler = ResumableUploadHandler()
382 self.empty_src_file.seek(0)
370 self.dst_key.set_contents_from_file( 383 self.dst_key.set_contents_from_file(
371 self.empty_src_file, res_upload_handler=res_upload_handler) 384 self.empty_src_file, res_upload_handler=res_upload_handler)
372 self.assertEqual(0, self.dst_key.size) 385 self.assertEqual(0, self.dst_key.size)
373 386
374 def test_upload_retains_metadata(self): 387 def test_upload_retains_metadata(self):
375 """ 388 """
376 Tests that resumable upload correctly sets passed metadata 389 Tests that resumable upload correctly sets passed metadata
377 """ 390 """
378 res_upload_handler = ResumableUploadHandler() 391 res_upload_handler = ResumableUploadHandler()
379 headers = {'Content-Type' : 'text/plain', 'Content-Encoding' : 'gzip', 392 headers = {'Content-Type' : 'text/plain', 'Content-Encoding' : 'gzip',
380 'x-goog-meta-abc' : 'my meta', 'x-goog-acl' : 'public-read'} 393 'x-goog-meta-abc' : 'my meta', 'x-goog-acl' : 'public-read'}
394 self.small_src_file.seek(0)
381 self.dst_key.set_contents_from_file( 395 self.dst_key.set_contents_from_file(
382 self.small_src_file, headers=headers, 396 self.small_src_file, headers=headers,
383 res_upload_handler=res_upload_handler) 397 res_upload_handler=res_upload_handler)
384 self.assertEqual(self.small_src_file_size, self.dst_key.size) 398 self.assertEqual(self.small_src_file_size, self.dst_key.size)
385 self.assertEqual(self.small_src_file_as_string, 399 self.assertEqual(self.small_src_file_as_string,
386 self.dst_key.get_contents_as_string()) 400 self.dst_key.get_contents_as_string())
387 self.dst_key.open_read() 401 self.dst_key.open_read()
388 self.assertEqual('text/plain', self.dst_key.content_type) 402 self.assertEqual('text/plain', self.dst_key.content_type)
389 self.assertEqual('gzip', self.dst_key.content_encoding) 403 self.assertEqual('gzip', self.dst_key.content_encoding)
390 self.assertTrue('abc' in self.dst_key.metadata) 404 self.assertTrue('abc' in self.dst_key.metadata)
(...skipping 10 matching lines...) Expand all
401 Tests resumable upload on a file that changes sizes between inital 415 Tests resumable upload on a file that changes sizes between inital
402 upload start and restart 416 upload start and restart
403 """ 417 """
404 harnass = CallbackTestHarnass( 418 harnass = CallbackTestHarnass(
405 fail_after_n_bytes=self.larger_src_file_size/2) 419 fail_after_n_bytes=self.larger_src_file_size/2)
406 # Set up first process' ResumableUploadHandler not to do any 420 # Set up first process' ResumableUploadHandler not to do any
407 # retries (initial upload request will establish expected size to 421 # retries (initial upload request will establish expected size to
408 # upload server). 422 # upload server).
409 res_upload_handler = ResumableUploadHandler( 423 res_upload_handler = ResumableUploadHandler(
410 tracker_file_name=self.tracker_file_name, num_retries=0) 424 tracker_file_name=self.tracker_file_name, num_retries=0)
425 self.larger_src_file.seek(0)
411 try: 426 try:
412 self.dst_key.set_contents_from_file( 427 self.dst_key.set_contents_from_file(
413 self.larger_src_file, cb=harnass.call, 428 self.larger_src_file, cb=harnass.call,
414 res_upload_handler=res_upload_handler) 429 res_upload_handler=res_upload_handler)
415 self.fail('Did not get expected ResumableUploadException') 430 self.fail('Did not get expected ResumableUploadException')
416 except ResumableUploadException, e: 431 except ResumableUploadException, e:
417 # First abort (from harnass-forced failure) should be 432 # First abort (from harnass-forced failure) should be
418 # ABORT_CUR_PROCESS. 433 # ABORT_CUR_PROCESS.
419 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_C UR_PROCESS) 434 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_C UR_PROCESS)
420 # Ensure a tracker file survived. 435 # Ensure a tracker file survived.
421 self.assertTrue(os.path.exists(self.tracker_file_name)) 436 self.assertTrue(os.path.exists(self.tracker_file_name))
422 # Try it again, this time with different size source file. 437 # Try it again, this time with different size source file.
423 # Wait 1 second between retry attempts, to give upload server a 438 # Wait 1 second between retry attempts, to give upload server a
424 # chance to save state so it can respond to changed file size with 439 # chance to save state so it can respond to changed file size with
425 # 500 response in the next attempt. 440 # 500 response in the next attempt.
426 time.sleep(1) 441 time.sleep(1)
427 try: 442 try:
443 self.largest_src_file.seek(0)
428 self.dst_key.set_contents_from_file( 444 self.dst_key.set_contents_from_file(
429 self.largest_src_file, res_upload_handler=res_upload_handler) 445 self.largest_src_file, res_upload_handler=res_upload_handler)
430 self.fail('Did not get expected ResumableUploadException') 446 self.fail('Did not get expected ResumableUploadException')
431 except ResumableUploadException, e: 447 except ResumableUploadException, e:
432 # This abort should be a hard abort (file size changing during 448 # This abort should be a hard abort (file size changing during
433 # transfer). 449 # transfer).
434 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) 450 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
435 self.assertNotEqual( 451 self.assertNotEqual(e.message.find('file size changed'), -1, e.messa ge)
436 e.message.find('attempt to upload a different size file'), -1)
437 452
438 def test_upload_with_file_size_change_during_upload(self): 453 def test_upload_with_file_size_change_during_upload(self):
439 """ 454 """
440 Tests resumable upload on a file that changes sizes while upload 455 Tests resumable upload on a file that changes sizes while upload
441 in progress 456 in progress
442 """ 457 """
443 # Create a file we can change during the upload. 458 # Create a file we can change during the upload.
444 test_file_size = 500 * 1024 # 500 KB. 459 test_file_size = 500 * 1024 # 500 KB.
445 test_file = self.build_test_input_file(test_file_size)[1] 460 test_file = self.build_test_input_file(test_file_size)[1]
446 harnass = CallbackTestHarnass(fp_to_change=test_file, 461 harnass = CallbackTestHarnass(fp_to_change=test_file,
(...skipping 29 matching lines...) Expand all
476 res_upload_handler=res_upload_handler) 491 res_upload_handler=res_upload_handler)
477 self.fail('Did not get expected ResumableUploadException') 492 self.fail('Did not get expected ResumableUploadException')
478 except ResumableUploadException, e: 493 except ResumableUploadException, e:
479 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) 494 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
480 # Ensure the file size didn't change. 495 # Ensure the file size didn't change.
481 test_file.seek(0, os.SEEK_END) 496 test_file.seek(0, os.SEEK_END)
482 self.assertEqual(test_file_size, test_file.tell()) 497 self.assertEqual(test_file_size, test_file.tell())
483 self.assertNotEqual( 498 self.assertNotEqual(
484 e.message.find('md5 signature doesn\'t match etag'), -1) 499 e.message.find('md5 signature doesn\'t match etag'), -1)
485 # Ensure the bad data wasn't left around. 500 # Ensure the bad data wasn't left around.
486 all_keys = self.dst_key_uri.get_all_keys() 501 try:
487 self.assertEqual(0, len(all_keys)) 502 self.dst_key_uri.get_key()
503 self.fail('Did not get expected InvalidUriError')
504 except InvalidUriError, e:
505 pass
488 506
489 def test_upload_with_content_length_header_set(self): 507 def test_upload_with_content_length_header_set(self):
490 """ 508 """
491 Tests resumable upload on a file when the user supplies a 509 Tests resumable upload on a file when the user supplies a
492 Content-Length header. This is used by gsutil, for example, 510 Content-Length header. This is used by gsutil, for example,
493 to set the content length when gzipping a file. 511 to set the content length when gzipping a file.
494 """ 512 """
495 res_upload_handler = ResumableUploadHandler() 513 res_upload_handler = ResumableUploadHandler()
514 self.small_src_file.seek(0)
496 try: 515 try:
497 self.dst_key.set_contents_from_file( 516 self.dst_key.set_contents_from_file(
498 self.small_src_file, res_upload_handler=res_upload_handler, 517 self.small_src_file, res_upload_handler=res_upload_handler,
499 headers={'Content-Length' : self.small_src_file_size}) 518 headers={'Content-Length' : self.small_src_file_size})
500 self.fail('Did not get expected ResumableUploadException') 519 self.fail('Did not get expected ResumableUploadException')
501 except ResumableUploadException, e: 520 except ResumableUploadException, e:
502 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) 521 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
503 self.assertNotEqual( 522 self.assertNotEqual(
504 e.message.find('Attempt to specify Content-Length header'), -1) 523 e.message.find('Attempt to specify Content-Length header'), -1)
505 524
506 def test_upload_with_syntactically_invalid_tracker_uri(self): 525 def test_upload_with_syntactically_invalid_tracker_uri(self):
507 """ 526 """
508 Tests resumable upload with a syntactically invalid tracker URI 527 Tests resumable upload with a syntactically invalid tracker URI
509 """ 528 """
510 res_upload_handler = ResumableUploadHandler( 529 res_upload_handler = ResumableUploadHandler(
511 tracker_file_name=self.syntactically_invalid_tracker_file_name) 530 tracker_file_name=self.syntactically_invalid_tracker_file_name)
512 # An error should be printed about the invalid URI, but then it 531 # An error should be printed about the invalid URI, but then it
513 # should run the update successfully. 532 # should run the update successfully.
533 self.small_src_file.seek(0)
514 self.dst_key.set_contents_from_file( 534 self.dst_key.set_contents_from_file(
515 self.small_src_file, res_upload_handler=res_upload_handler) 535 self.small_src_file, res_upload_handler=res_upload_handler)
516 self.assertEqual(self.small_src_file_size, self.dst_key.size) 536 self.assertEqual(self.small_src_file_size, self.dst_key.size)
517 self.assertEqual(self.small_src_file_as_string, 537 self.assertEqual(self.small_src_file_as_string,
518 self.dst_key.get_contents_as_string()) 538 self.dst_key.get_contents_as_string())
519 539
520 def test_upload_with_invalid_upload_id_in_tracker_file(self): 540 def test_upload_with_invalid_upload_id_in_tracker_file(self):
521 """ 541 """
522 Tests resumable upload with invalid upload ID 542 Tests resumable upload with invalid upload ID
523 """ 543 """
524 res_upload_handler = ResumableUploadHandler( 544 res_upload_handler = ResumableUploadHandler(
525 tracker_file_name=self.invalid_upload_id_tracker_file_name) 545 tracker_file_name=self.invalid_upload_id_tracker_file_name)
526 # An error should occur, but then the tracker URI should be 546 # An error should occur, but then the tracker URI should be
527 # regenerated and the the update should succeed. 547 # regenerated and the the update should succeed.
548 self.small_src_file.seek(0)
528 self.dst_key.set_contents_from_file( 549 self.dst_key.set_contents_from_file(
529 self.small_src_file, res_upload_handler=res_upload_handler) 550 self.small_src_file, res_upload_handler=res_upload_handler)
530 self.assertEqual(self.small_src_file_size, self.dst_key.size) 551 self.assertEqual(self.small_src_file_size, self.dst_key.size)
531 self.assertEqual(self.small_src_file_as_string, 552 self.assertEqual(self.small_src_file_as_string,
532 self.dst_key.get_contents_as_string()) 553 self.dst_key.get_contents_as_string())
533 self.assertNotEqual(self.invalid_upload_id, 554 self.assertNotEqual(self.invalid_upload_id,
534 res_upload_handler.get_tracker_uri()) 555 res_upload_handler.get_tracker_uri())
535 556
536 def test_upload_with_unwritable_tracker_file(self): 557 def test_upload_with_unwritable_tracker_file(self):
537 """ 558 """
538 Tests resumable upload with an unwritable tracker file 559 Tests resumable upload with an unwritable tracker file
539 """ 560 """
540 # Make dir where tracker_file lives temporarily unwritable. 561 # Make dir where tracker_file lives temporarily unwritable.
541 save_mod = os.stat(self.tmp_dir).st_mode 562 save_mod = os.stat(self.tmp_dir).st_mode
542 try: 563 try:
543 os.chmod(self.tmp_dir, 0) 564 os.chmod(self.tmp_dir, 0)
544 res_upload_handler = ResumableUploadHandler( 565 res_upload_handler = ResumableUploadHandler(
545 tracker_file_name=self.tracker_file_name) 566 tracker_file_name=self.tracker_file_name)
546 except ResumableUploadException, e: 567 except ResumableUploadException, e:
547 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) 568 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
548 self.assertNotEqual( 569 self.assertNotEqual(
549 e.message.find('Couldn\'t write URI tracker file'), -1) 570 e.message.find('Couldn\'t write URI tracker file'), -1)
550 finally: 571 finally:
551 # Restore original protection of dir where tracker_file lives. 572 # Restore original protection of dir where tracker_file lives.
552 os.chmod(self.tmp_dir, save_mod) 573 os.chmod(self.tmp_dir, save_mod)
553
554 if __name__ == '__main__':
555 if sys.version_info[:3] < (2, 5, 1):
556 sys.exit('These tests must be run on at least Python 2.5.1\n')
557
558 # Use -d to see more HTTP protocol detail during tests.
559 debug = 0
560 opts, args = getopt.getopt(sys.argv[1:], 'd', ['debug'])
561 for o, a in opts:
562 if o in ('-d', '--debug'):
563 debug = 2
564
565 test_loader = unittest.TestLoader()
566 test_loader.testMethodPrefix = 'test_'
567 suite = test_loader.loadTestsFromTestCase(ResumableUploadTests)
568 # Seems like there should be a cleaner way to find the test_class.
569 test_class = suite.__getattribute__('_tests')[0]
570 # We call set_up_class() and tear_down_class() ourselves because we
571 # don't assume the user has Python 2.7 (which supports classmethods
572 # that do it, with camelCase versions of these names).
573 try:
574 print 'Setting up %s...' % test_class.get_suite_description()
575 test_class.set_up_class(debug)
576 print 'Running %s...' % test_class.get_suite_description()
577 unittest.TextTestRunner(verbosity=2).run(suite)
578 finally:
579 print 'Cleaning up after %s...' % test_class.get_suite_description()
580 test_class.tear_down_class()
581 print ''
OLDNEW
« no previous file with comments | « third_party/gsutil/boto/tests/s3/test_resumable_downloads.py ('k') | third_party/gsutil/boto/tests/s3/test_versioning.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698