| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2010 Google Inc. | |
| 4 # | |
| 5 # Permission is hereby granted, free of charge, to any person obtaining a | |
| 6 # copy of this software and associated documentation files (the | |
| 7 # "Software"), to deal in the Software without restriction, including | |
| 8 # without limitation the rights to use, copy, modify, merge, publish, dis- | |
| 9 # tribute, sublicense, and/or sell copies of the Software, and to permit | |
| 10 # persons to whom the Software is furnished to do so, subject to the fol- | |
| 11 # lowing conditions: | |
| 12 # | |
| 13 # The above copyright notice and this permission notice shall be included | |
| 14 # in all copies or substantial portions of the Software. | |
| 15 # | |
| 16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS | |
| 17 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- | |
| 18 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT | |
| 19 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, | |
| 20 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| 21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| 22 # IN THE SOFTWARE. | |
| 23 | |
| 24 """ | |
| 25 Tests of Google Storage resumable uploads. | |
| 26 """ | |
| 27 | |
| 28 import errno | |
| 29 import getopt | |
| 30 import os | |
| 31 import random | |
| 32 import re | |
| 33 import shutil | |
| 34 import socket | |
| 35 import StringIO | |
| 36 import sys | |
| 37 import tempfile | |
| 38 import time | |
| 39 import unittest | |
| 40 | |
| 41 import boto | |
| 42 from boto.exception import GSResponseError | |
| 43 from boto.gs.resumable_upload_handler import ResumableUploadHandler | |
| 44 from boto.exception import ResumableTransferDisposition | |
| 45 from boto.exception import ResumableUploadException | |
| 46 from boto.exception import StorageResponseError | |
| 47 from tests.s3.cb_test_harnass import CallbackTestHarnass | |
| 48 | |
| 49 # We don't use the OAuth2 authentication plugin directly; importing it here | |
| 50 # ensures that it's loaded and available by default. | |
| 51 try: | |
| 52 from oauth2_plugin import oauth2_plugin | |
| 53 except ImportError: | |
| 54 # Do nothing - if user doesn't have OAuth2 configured it doesn't matter; | |
| 55 # and if they do, the tests will fail (as they should in that case). | |
| 56 pass | |
| 57 | |
| 58 | |
| 59 class ResumableUploadTests(unittest.TestCase): | |
| 60 """ | |
| 61 Resumable upload test suite. | |
| 62 """ | |
| 63 | |
| 64 def get_suite_description(self): | |
| 65 return 'Resumable upload test suite' | |
| 66 | |
| 67 @classmethod | |
| 68 def setUp(cls): | |
| 69 """ | |
| 70 Creates dst_key needed by all tests. | |
| 71 | |
| 72 This method's namingCase is required by the unittest framework. | |
| 73 """ | |
| 74 cls.dst_key = cls.dst_key_uri.new_key(validate=False) | |
| 75 | |
| 76 @classmethod | |
| 77 def tearDown(cls): | |
| 78 """ | |
| 79 Deletes any objects or files created by last test run. | |
| 80 | |
| 81 This method's namingCase is required by the unittest framework. | |
| 82 """ | |
| 83 try: | |
| 84 cls.dst_key_uri.delete_key() | |
| 85 except GSResponseError: | |
| 86 # Ignore possible not-found error. | |
| 87 pass | |
| 88 # Recursively delete dst dir and then re-create it, so in effect we | |
| 89 # remove all dirs and files under that directory. | |
| 90 shutil.rmtree(cls.tmp_dir) | |
| 91 os.mkdir(cls.tmp_dir) | |
| 92 | |
| 93 @staticmethod | |
| 94 def build_test_input_file(size): | |
| 95 buf = [] | |
| 96 # I manually construct the random data here instead of calling | |
| 97 # os.urandom() because I want to constrain the range of data (in | |
| 98 # this case to 0'..'9') so the test | |
| 99 # code can easily overwrite part of the StringIO file with | |
| 100 # known-to-be-different values. | |
| 101 for i in range(size): | |
| 102 buf.append(str(random.randint(0, 9))) | |
| 103 file_as_string = ''.join(buf) | |
| 104 return (file_as_string, StringIO.StringIO(file_as_string)) | |
| 105 | |
| 106 @classmethod | |
| 107 def set_up_class(cls, debug): | |
| 108 """ | |
| 109 Initializes test suite. | |
| 110 """ | |
| 111 | |
| 112 # Use a designated tmpdir prefix to make it easy to find the end of | |
| 113 # the tmp path. | |
| 114 cls.tmpdir_prefix = 'tmp_resumable_upload_test' | |
| 115 | |
| 116 # Create test source file data. | |
| 117 cls.empty_src_file_size = 0 | |
| 118 (cls.empty_src_file_as_string, cls.empty_src_file) = ( | |
| 119 cls.build_test_input_file(cls.empty_src_file_size)) | |
| 120 cls.small_src_file_size = 2 * 1024 # 2 KB. | |
| 121 (cls.small_src_file_as_string, cls.small_src_file) = ( | |
| 122 cls.build_test_input_file(cls.small_src_file_size)) | |
| 123 cls.larger_src_file_size = 500 * 1024 # 500 KB. | |
| 124 (cls.larger_src_file_as_string, cls.larger_src_file) = ( | |
| 125 cls.build_test_input_file(cls.larger_src_file_size)) | |
| 126 cls.largest_src_file_size = 1024 * 1024 # 1 MB. | |
| 127 (cls.largest_src_file_as_string, cls.largest_src_file) = ( | |
| 128 cls.build_test_input_file(cls.largest_src_file_size)) | |
| 129 | |
| 130 # Create temp dir. | |
| 131 cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix) | |
| 132 | |
| 133 # Create the test bucket. | |
| 134 hostname = socket.gethostname().split('.')[0] | |
| 135 cls.uri_base_str = 'gs://res_upload_test_%s_%s_%s' % ( | |
| 136 hostname, os.getpid(), int(time.time())) | |
| 137 cls.dst_bucket_uri = boto.storage_uri('%s_dst' % | |
| 138 cls.uri_base_str, debug=debug) | |
| 139 cls.dst_bucket_uri.create_bucket() | |
| 140 cls.dst_key_uri = cls.dst_bucket_uri.clone_replace_name('obj') | |
| 141 | |
| 142 cls.tracker_file_name = '%s%suri_tracker' % (cls.tmp_dir, os.sep) | |
| 143 | |
| 144 cls.syntactically_invalid_tracker_file_name = ( | |
| 145 '%s%ssynt_invalid_uri_tracker' % (cls.tmp_dir, os.sep)) | |
| 146 f = open(cls.syntactically_invalid_tracker_file_name, 'w') | |
| 147 f.write('ftp://example.com') | |
| 148 f.close() | |
| 149 | |
| 150 cls.invalid_upload_id = ( | |
| 151 'http://pub.commondatastorage.googleapis.com/?upload_id=' | |
| 152 'AyzB2Uo74W4EYxyi5dp_-r68jz8rtbvshsv4TX7srJVkJ57CxTY5Dw2') | |
| 153 cls.invalid_upload_id_tracker_file_name = ( | |
| 154 '%s%sinvalid_upload_id_tracker' % (cls.tmp_dir, os.sep)) | |
| 155 f = open(cls.invalid_upload_id_tracker_file_name, 'w') | |
| 156 f.write(cls.invalid_upload_id) | |
| 157 f.close() | |
| 158 | |
| 159 cls.created_test_data = True | |
| 160 | |
| 161 @classmethod | |
| 162 def tear_down_class(cls): | |
| 163 """ | |
| 164 Deletes bucket and tmp dir created by set_up_class. | |
| 165 """ | |
| 166 if not hasattr(cls, 'created_test_data'): | |
| 167 return | |
| 168 # Call cls.tearDown() in case the tests got interrupted, to ensure | |
| 169 # dst objects get deleted. | |
| 170 cls.tearDown() | |
| 171 | |
| 172 # Retry (for up to 2 minutes) the bucket gets deleted (it may not | |
| 173 # the first time round, due to eventual consistency of bucket delete | |
| 174 # operations). | |
| 175 for i in range(60): | |
| 176 try: | |
| 177 cls.dst_bucket_uri.delete_bucket() | |
| 178 break | |
| 179 except StorageResponseError: | |
| 180 print 'Test bucket (%s) not yet deleted, still trying' % ( | |
| 181 cls.dst_bucket_uri.uri) | |
| 182 time.sleep(2) | |
| 183 shutil.rmtree(cls.tmp_dir) | |
| 184 cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix) | |
| 185 | |
| 186 def test_non_resumable_upload(self): | |
| 187 """ | |
| 188 Tests that non-resumable uploads work | |
| 189 """ | |
| 190 self.dst_key.set_contents_from_file(self.small_src_file) | |
| 191 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 192 self.assertEqual(self.small_src_file_as_string, | |
| 193 self.dst_key.get_contents_as_string()) | |
| 194 | |
| 195 def test_upload_without_persistent_tracker(self): | |
| 196 """ | |
| 197 Tests a single resumable upload, with no tracker URI persistence | |
| 198 """ | |
| 199 res_upload_handler = ResumableUploadHandler() | |
| 200 self.dst_key.set_contents_from_file( | |
| 201 self.small_src_file, res_upload_handler=res_upload_handler) | |
| 202 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 203 self.assertEqual(self.small_src_file_as_string, | |
| 204 self.dst_key.get_contents_as_string()) | |
| 205 | |
| 206 def test_failed_upload_with_persistent_tracker(self): | |
| 207 """ | |
| 208 Tests that failed resumable upload leaves a correct tracker URI file | |
| 209 """ | |
| 210 harnass = CallbackTestHarnass() | |
| 211 res_upload_handler = ResumableUploadHandler( | |
| 212 tracker_file_name=self.tracker_file_name, num_retries=0) | |
| 213 try: | |
| 214 self.dst_key.set_contents_from_file( | |
| 215 self.small_src_file, cb=harnass.call, | |
| 216 res_upload_handler=res_upload_handler) | |
| 217 self.fail('Did not get expected ResumableUploadException') | |
| 218 except ResumableUploadException, e: | |
| 219 # We'll get a ResumableUploadException at this point because | |
| 220 # of CallbackTestHarnass (above). Check that the tracker file was | |
| 221 # created correctly. | |
| 222 self.assertEqual(e.disposition, | |
| 223 ResumableTransferDisposition.ABORT_CUR_PROCESS) | |
| 224 self.assertTrue(os.path.exists(self.tracker_file_name)) | |
| 225 f = open(self.tracker_file_name) | |
| 226 uri_from_file = f.readline().strip() | |
| 227 f.close() | |
| 228 self.assertEqual(uri_from_file, | |
| 229 res_upload_handler.get_tracker_uri()) | |
| 230 | |
| 231 def test_retryable_exception_recovery(self): | |
| 232 """ | |
| 233 Tests handling of a retryable exception | |
| 234 """ | |
| 235 # Test one of the RETRYABLE_EXCEPTIONS. | |
| 236 exception = ResumableUploadHandler.RETRYABLE_EXCEPTIONS[0] | |
| 237 harnass = CallbackTestHarnass(exception=exception) | |
| 238 res_upload_handler = ResumableUploadHandler(num_retries=1) | |
| 239 self.dst_key.set_contents_from_file( | |
| 240 self.small_src_file, cb=harnass.call, | |
| 241 res_upload_handler=res_upload_handler) | |
| 242 # Ensure uploaded object has correct content. | |
| 243 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 244 self.assertEqual(self.small_src_file_as_string, | |
| 245 self.dst_key.get_contents_as_string()) | |
| 246 | |
| 247 def test_broken_pipe_recovery(self): | |
| 248 """ | |
| 249 Tests handling of a Broken Pipe (which interacts with an httplib bug) | |
| 250 """ | |
| 251 exception = IOError(errno.EPIPE, "Broken pipe") | |
| 252 harnass = CallbackTestHarnass(exception=exception) | |
| 253 res_upload_handler = ResumableUploadHandler(num_retries=1) | |
| 254 self.dst_key.set_contents_from_file( | |
| 255 self.small_src_file, cb=harnass.call, | |
| 256 res_upload_handler=res_upload_handler) | |
| 257 # Ensure uploaded object has correct content. | |
| 258 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 259 self.assertEqual(self.small_src_file_as_string, | |
| 260 self.dst_key.get_contents_as_string()) | |
| 261 | |
| 262 def test_non_retryable_exception_handling(self): | |
| 263 """ | |
| 264 Tests a resumable upload that fails with a non-retryable exception | |
| 265 """ | |
| 266 harnass = CallbackTestHarnass( | |
| 267 exception=OSError(errno.EACCES, 'Permission denied')) | |
| 268 res_upload_handler = ResumableUploadHandler(num_retries=1) | |
| 269 try: | |
| 270 self.dst_key.set_contents_from_file( | |
| 271 self.small_src_file, cb=harnass.call, | |
| 272 res_upload_handler=res_upload_handler) | |
| 273 self.fail('Did not get expected OSError') | |
| 274 except OSError, e: | |
| 275 # Ensure the error was re-raised. | |
| 276 self.assertEqual(e.errno, 13) | |
| 277 | |
| 278 def test_failed_and_restarted_upload_with_persistent_tracker(self): | |
| 279 """ | |
| 280 Tests resumable upload that fails once and then completes, with tracker | |
| 281 file | |
| 282 """ | |
| 283 harnass = CallbackTestHarnass() | |
| 284 res_upload_handler = ResumableUploadHandler( | |
| 285 tracker_file_name=self.tracker_file_name, num_retries=1) | |
| 286 self.dst_key.set_contents_from_file( | |
| 287 self.small_src_file, cb=harnass.call, | |
| 288 res_upload_handler=res_upload_handler) | |
| 289 # Ensure uploaded object has correct content. | |
| 290 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 291 self.assertEqual(self.small_src_file_as_string, | |
| 292 self.dst_key.get_contents_as_string()) | |
| 293 # Ensure tracker file deleted. | |
| 294 self.assertFalse(os.path.exists(self.tracker_file_name)) | |
| 295 | |
| 296 def test_multiple_in_process_failures_then_succeed(self): | |
| 297 """ | |
| 298 Tests resumable upload that fails twice in one process, then completes | |
| 299 """ | |
| 300 res_upload_handler = ResumableUploadHandler(num_retries=3) | |
| 301 self.dst_key.set_contents_from_file( | |
| 302 self.small_src_file, res_upload_handler=res_upload_handler) | |
| 303 # Ensure uploaded object has correct content. | |
| 304 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 305 self.assertEqual(self.small_src_file_as_string, | |
| 306 self.dst_key.get_contents_as_string()) | |
| 307 | |
| 308 def test_multiple_in_process_failures_then_succeed_with_tracker_file(self): | |
| 309 """ | |
| 310 Tests resumable upload that fails completely in one process, | |
| 311 then when restarted completes, using a tracker file | |
| 312 """ | |
| 313 # Set up test harnass that causes more failures than a single | |
| 314 # ResumableUploadHandler instance will handle, writing enough data | |
| 315 # before the first failure that some of it survives that process run. | |
| 316 harnass = CallbackTestHarnass( | |
| 317 fail_after_n_bytes=self.larger_src_file_size/2, num_times_to_fail=2) | |
| 318 res_upload_handler = ResumableUploadHandler( | |
| 319 tracker_file_name=self.tracker_file_name, num_retries=1) | |
| 320 try: | |
| 321 self.dst_key.set_contents_from_file( | |
| 322 self.larger_src_file, cb=harnass.call, | |
| 323 res_upload_handler=res_upload_handler) | |
| 324 self.fail('Did not get expected ResumableUploadException') | |
| 325 except ResumableUploadException, e: | |
| 326 self.assertEqual(e.disposition, | |
| 327 ResumableTransferDisposition.ABORT_CUR_PROCESS) | |
| 328 # Ensure a tracker file survived. | |
| 329 self.assertTrue(os.path.exists(self.tracker_file_name)) | |
| 330 # Try it one more time; this time should succeed. | |
| 331 self.dst_key.set_contents_from_file( | |
| 332 self.larger_src_file, cb=harnass.call, | |
| 333 res_upload_handler=res_upload_handler) | |
| 334 self.assertEqual(self.larger_src_file_size, self.dst_key.size) | |
| 335 self.assertEqual(self.larger_src_file_as_string, | |
| 336 self.dst_key.get_contents_as_string()) | |
| 337 self.assertFalse(os.path.exists(self.tracker_file_name)) | |
| 338 # Ensure some of the file was uploaded both before and after failure. | |
| 339 self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1 | |
| 340 and | |
| 341 len(harnass.transferred_seq_after_first_failure) > 1) | |
| 342 | |
| 343 def test_upload_with_inital_partial_upload_before_failure(self): | |
| 344 """ | |
| 345 Tests resumable upload that successfully uploads some content | |
| 346 before it fails, then restarts and completes | |
| 347 """ | |
| 348 # Set up harnass to fail upload after several hundred KB so upload | |
| 349 # server will have saved something before we retry. | |
| 350 harnass = CallbackTestHarnass( | |
| 351 fail_after_n_bytes=self.larger_src_file_size/2) | |
| 352 res_upload_handler = ResumableUploadHandler(num_retries=1) | |
| 353 self.dst_key.set_contents_from_file( | |
| 354 self.larger_src_file, cb=harnass.call, | |
| 355 res_upload_handler=res_upload_handler) | |
| 356 # Ensure uploaded object has correct content. | |
| 357 self.assertEqual(self.larger_src_file_size, self.dst_key.size) | |
| 358 self.assertEqual(self.larger_src_file_as_string, | |
| 359 self.dst_key.get_contents_as_string()) | |
| 360 # Ensure some of the file was uploaded both before and after failure. | |
| 361 self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1 | |
| 362 and | |
| 363 len(harnass.transferred_seq_after_first_failure) > 1) | |
| 364 | |
| 365 def test_empty_file_upload(self): | |
| 366 """ | |
| 367 Tests uploading an empty file (exercises boundary conditions). | |
| 368 """ | |
| 369 res_upload_handler = ResumableUploadHandler() | |
| 370 self.dst_key.set_contents_from_file( | |
| 371 self.empty_src_file, res_upload_handler=res_upload_handler) | |
| 372 self.assertEqual(0, self.dst_key.size) | |
| 373 | |
| 374 def test_upload_retains_metadata(self): | |
| 375 """ | |
| 376 Tests that resumable upload correctly sets passed metadata | |
| 377 """ | |
| 378 res_upload_handler = ResumableUploadHandler() | |
| 379 headers = {'Content-Type' : 'text/plain', 'Content-Encoding' : 'gzip', | |
| 380 'x-goog-meta-abc' : 'my meta', 'x-goog-acl' : 'public-read'} | |
| 381 self.dst_key.set_contents_from_file( | |
| 382 self.small_src_file, headers=headers, | |
| 383 res_upload_handler=res_upload_handler) | |
| 384 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 385 self.assertEqual(self.small_src_file_as_string, | |
| 386 self.dst_key.get_contents_as_string()) | |
| 387 self.dst_key.open_read() | |
| 388 self.assertEqual('text/plain', self.dst_key.content_type) | |
| 389 self.assertEqual('gzip', self.dst_key.content_encoding) | |
| 390 self.assertTrue('abc' in self.dst_key.metadata) | |
| 391 self.assertEqual('my meta', str(self.dst_key.metadata['abc'])) | |
| 392 acl = self.dst_key.get_acl() | |
| 393 for entry in acl.entries.entry_list: | |
| 394 if str(entry.scope) == '<AllUsers>': | |
| 395 self.assertEqual('READ', str(acl.entries.entry_list[1].permissio
n)) | |
| 396 return | |
| 397 self.fail('No <AllUsers> scope found') | |
| 398 | |
| 399 def test_upload_with_file_size_change_between_starts(self): | |
| 400 """ | |
| 401 Tests resumable upload on a file that changes sizes between inital | |
| 402 upload start and restart | |
| 403 """ | |
| 404 harnass = CallbackTestHarnass( | |
| 405 fail_after_n_bytes=self.larger_src_file_size/2) | |
| 406 # Set up first process' ResumableUploadHandler not to do any | |
| 407 # retries (initial upload request will establish expected size to | |
| 408 # upload server). | |
| 409 res_upload_handler = ResumableUploadHandler( | |
| 410 tracker_file_name=self.tracker_file_name, num_retries=0) | |
| 411 try: | |
| 412 self.dst_key.set_contents_from_file( | |
| 413 self.larger_src_file, cb=harnass.call, | |
| 414 res_upload_handler=res_upload_handler) | |
| 415 self.fail('Did not get expected ResumableUploadException') | |
| 416 except ResumableUploadException, e: | |
| 417 # First abort (from harnass-forced failure) should be | |
| 418 # ABORT_CUR_PROCESS. | |
| 419 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_C
UR_PROCESS) | |
| 420 # Ensure a tracker file survived. | |
| 421 self.assertTrue(os.path.exists(self.tracker_file_name)) | |
| 422 # Try it again, this time with different size source file. | |
| 423 # Wait 1 second between retry attempts, to give upload server a | |
| 424 # chance to save state so it can respond to changed file size with | |
| 425 # 500 response in the next attempt. | |
| 426 time.sleep(1) | |
| 427 try: | |
| 428 self.dst_key.set_contents_from_file( | |
| 429 self.largest_src_file, res_upload_handler=res_upload_handler) | |
| 430 self.fail('Did not get expected ResumableUploadException') | |
| 431 except ResumableUploadException, e: | |
| 432 # This abort should be a hard abort (file size changing during | |
| 433 # transfer). | |
| 434 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 435 self.assertNotEqual( | |
| 436 e.message.find('attempt to upload a different size file'), -1) | |
| 437 | |
| 438 def test_upload_with_file_size_change_during_upload(self): | |
| 439 """ | |
| 440 Tests resumable upload on a file that changes sizes while upload | |
| 441 in progress | |
| 442 """ | |
| 443 # Create a file we can change during the upload. | |
| 444 test_file_size = 500 * 1024 # 500 KB. | |
| 445 test_file = self.build_test_input_file(test_file_size)[1] | |
| 446 harnass = CallbackTestHarnass(fp_to_change=test_file, | |
| 447 fp_change_pos=test_file_size) | |
| 448 res_upload_handler = ResumableUploadHandler(num_retries=1) | |
| 449 try: | |
| 450 self.dst_key.set_contents_from_file( | |
| 451 test_file, cb=harnass.call, | |
| 452 res_upload_handler=res_upload_handler) | |
| 453 self.fail('Did not get expected ResumableUploadException') | |
| 454 except ResumableUploadException, e: | |
| 455 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 456 self.assertNotEqual( | |
| 457 e.message.find('File changed during upload'), -1) | |
| 458 | |
| 459 def test_upload_with_file_content_change_during_upload(self): | |
| 460 """ | |
| 461 Tests resumable upload on a file that changes one byte of content | |
| 462 (so, size stays the same) while upload in progress | |
| 463 """ | |
| 464 test_file_size = 500 * 1024 # 500 KB. | |
| 465 test_file = self.build_test_input_file(test_file_size)[1] | |
| 466 harnass = CallbackTestHarnass(fail_after_n_bytes=test_file_size/2, | |
| 467 fp_to_change=test_file, | |
| 468 # Writing at file_size-5 won't change file | |
| 469 # size because CallbackTestHarnass only | |
| 470 # writes 3 bytes. | |
| 471 fp_change_pos=test_file_size-5) | |
| 472 res_upload_handler = ResumableUploadHandler(num_retries=1) | |
| 473 try: | |
| 474 self.dst_key.set_contents_from_file( | |
| 475 test_file, cb=harnass.call, | |
| 476 res_upload_handler=res_upload_handler) | |
| 477 self.fail('Did not get expected ResumableUploadException') | |
| 478 except ResumableUploadException, e: | |
| 479 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 480 # Ensure the file size didn't change. | |
| 481 test_file.seek(0, os.SEEK_END) | |
| 482 self.assertEqual(test_file_size, test_file.tell()) | |
| 483 self.assertNotEqual( | |
| 484 e.message.find('md5 signature doesn\'t match etag'), -1) | |
| 485 # Ensure the bad data wasn't left around. | |
| 486 all_keys = self.dst_key_uri.get_all_keys() | |
| 487 self.assertEqual(0, len(all_keys)) | |
| 488 | |
| 489 def test_upload_with_content_length_header_set(self): | |
| 490 """ | |
| 491 Tests resumable upload on a file when the user supplies a | |
| 492 Content-Length header. This is used by gsutil, for example, | |
| 493 to set the content length when gzipping a file. | |
| 494 """ | |
| 495 res_upload_handler = ResumableUploadHandler() | |
| 496 try: | |
| 497 self.dst_key.set_contents_from_file( | |
| 498 self.small_src_file, res_upload_handler=res_upload_handler, | |
| 499 headers={'Content-Length' : self.small_src_file_size}) | |
| 500 self.fail('Did not get expected ResumableUploadException') | |
| 501 except ResumableUploadException, e: | |
| 502 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 503 self.assertNotEqual( | |
| 504 e.message.find('Attempt to specify Content-Length header'), -1) | |
| 505 | |
| 506 def test_upload_with_syntactically_invalid_tracker_uri(self): | |
| 507 """ | |
| 508 Tests resumable upload with a syntactically invalid tracker URI | |
| 509 """ | |
| 510 res_upload_handler = ResumableUploadHandler( | |
| 511 tracker_file_name=self.syntactically_invalid_tracker_file_name) | |
| 512 # An error should be printed about the invalid URI, but then it | |
| 513 # should run the update successfully. | |
| 514 self.dst_key.set_contents_from_file( | |
| 515 self.small_src_file, res_upload_handler=res_upload_handler) | |
| 516 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 517 self.assertEqual(self.small_src_file_as_string, | |
| 518 self.dst_key.get_contents_as_string()) | |
| 519 | |
| 520 def test_upload_with_invalid_upload_id_in_tracker_file(self): | |
| 521 """ | |
| 522 Tests resumable upload with invalid upload ID | |
| 523 """ | |
| 524 res_upload_handler = ResumableUploadHandler( | |
| 525 tracker_file_name=self.invalid_upload_id_tracker_file_name) | |
| 526 # An error should occur, but then the tracker URI should be | |
| 527 # regenerated and the the update should succeed. | |
| 528 self.dst_key.set_contents_from_file( | |
| 529 self.small_src_file, res_upload_handler=res_upload_handler) | |
| 530 self.assertEqual(self.small_src_file_size, self.dst_key.size) | |
| 531 self.assertEqual(self.small_src_file_as_string, | |
| 532 self.dst_key.get_contents_as_string()) | |
| 533 self.assertNotEqual(self.invalid_upload_id, | |
| 534 res_upload_handler.get_tracker_uri()) | |
| 535 | |
| 536 def test_upload_with_unwritable_tracker_file(self): | |
| 537 """ | |
| 538 Tests resumable upload with an unwritable tracker file | |
| 539 """ | |
| 540 # Make dir where tracker_file lives temporarily unwritable. | |
| 541 save_mod = os.stat(self.tmp_dir).st_mode | |
| 542 try: | |
| 543 os.chmod(self.tmp_dir, 0) | |
| 544 res_upload_handler = ResumableUploadHandler( | |
| 545 tracker_file_name=self.tracker_file_name) | |
| 546 except ResumableUploadException, e: | |
| 547 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 548 self.assertNotEqual( | |
| 549 e.message.find('Couldn\'t write URI tracker file'), -1) | |
| 550 finally: | |
| 551 # Restore original protection of dir where tracker_file lives. | |
| 552 os.chmod(self.tmp_dir, save_mod) | |
| 553 | |
| 554 if __name__ == '__main__': | |
| 555 if sys.version_info[:3] < (2, 5, 1): | |
| 556 sys.exit('These tests must be run on at least Python 2.5.1\n') | |
| 557 | |
| 558 # Use -d to see more HTTP protocol detail during tests. | |
| 559 debug = 0 | |
| 560 opts, args = getopt.getopt(sys.argv[1:], 'd', ['debug']) | |
| 561 for o, a in opts: | |
| 562 if o in ('-d', '--debug'): | |
| 563 debug = 2 | |
| 564 | |
| 565 test_loader = unittest.TestLoader() | |
| 566 test_loader.testMethodPrefix = 'test_' | |
| 567 suite = test_loader.loadTestsFromTestCase(ResumableUploadTests) | |
| 568 # Seems like there should be a cleaner way to find the test_class. | |
| 569 test_class = suite.__getattribute__('_tests')[0] | |
| 570 # We call set_up_class() and tear_down_class() ourselves because we | |
| 571 # don't assume the user has Python 2.7 (which supports classmethods | |
| 572 # that do it, with camelCase versions of these names). | |
| 573 try: | |
| 574 print 'Setting up %s...' % test_class.get_suite_description() | |
| 575 test_class.set_up_class(debug) | |
| 576 print 'Running %s...' % test_class.get_suite_description() | |
| 577 unittest.TextTestRunner(verbosity=2).run(suite) | |
| 578 finally: | |
| 579 print 'Cleaning up after %s...' % test_class.get_suite_description() | |
| 580 test_class.tear_down_class() | |
| 581 print '' | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 10199002:
Upgrade gsutil to 3.4 (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart