| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2010 Google Inc. | |
| 4 # | |
| 5 # Permission is hereby granted, free of charge, to any person obtaining a | |
| 6 # copy of this software and associated documentation files (the | |
| 7 # "Software"), to deal in the Software without restriction, including | |
| 8 # without limitation the rights to use, copy, modify, merge, publish, dis- | |
| 9 # tribute, sublicense, and/or sell copies of the Software, and to permit | |
| 10 # persons to whom the Software is furnished to do so, subject to the fol- | |
| 11 # lowing conditions: | |
| 12 # | |
| 13 # The above copyright notice and this permission notice shall be included | |
| 14 # in all copies or substantial portions of the Software. | |
| 15 # | |
| 16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS | |
| 17 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- | |
| 18 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT | |
| 19 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, | |
| 20 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| 21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| 22 # IN THE SOFTWARE. | |
| 23 | |
| 24 """ | |
| 25 Tests of resumable downloads. | |
| 26 """ | |
| 27 | |
| 28 import errno | |
| 29 import getopt | |
| 30 import os | |
| 31 import random | |
| 32 import re | |
| 33 import shutil | |
| 34 import socket | |
| 35 import StringIO | |
| 36 import sys | |
| 37 import tempfile | |
| 38 import time | |
| 39 import unittest | |
| 40 | |
| 41 import boto | |
| 42 from boto import storage_uri | |
| 43 from boto.s3.resumable_download_handler import get_cur_file_size | |
| 44 from boto.s3.resumable_download_handler import ResumableDownloadHandler | |
| 45 from boto.exception import ResumableTransferDisposition | |
| 46 from boto.exception import ResumableDownloadException | |
| 47 from boto.exception import StorageResponseError | |
| 48 from tests.s3.cb_test_harnass import CallbackTestHarnass | |
| 49 | |
| 50 # We don't use the OAuth2 authentication plugin directly; importing it here | |
| 51 # ensures that it's loaded and available by default. | |
| 52 try: | |
| 53 from oauth2_plugin import oauth2_plugin | |
| 54 except ImportError: | |
| 55 # Do nothing - if user doesn't have OAuth2 configured it doesn't matter; | |
| 56 # and if they do, the tests will fail (as they should in that case). | |
| 57 pass | |
| 58 | |
| 59 | |
| 60 class ResumableDownloadTests(unittest.TestCase): | |
| 61 """ | |
| 62 Resumable download test suite. | |
| 63 """ | |
| 64 | |
| 65 def get_suite_description(self): | |
| 66 return 'Resumable download test suite' | |
| 67 | |
| 68 @staticmethod | |
| 69 def resilient_close(key): | |
| 70 try: | |
| 71 key.close() | |
| 72 except StorageResponseError, e: | |
| 73 pass | |
| 74 | |
| 75 @classmethod | |
| 76 def setUp(cls): | |
| 77 """ | |
| 78 Creates file-like object for detination of each download test. | |
| 79 | |
| 80 This method's namingCase is required by the unittest framework. | |
| 81 """ | |
| 82 cls.dst_fp = open(cls.dst_file_name, 'w') | |
| 83 | |
| 84 @classmethod | |
| 85 def tearDown(cls): | |
| 86 """ | |
| 87 Deletes any objects or files created by last test run, and closes | |
| 88 any keys in case they were read incompletely (which would leave | |
| 89 partial buffers of data for subsequent tests to trip over). | |
| 90 | |
| 91 This method's namingCase is required by the unittest framework. | |
| 92 """ | |
| 93 # Recursively delete dst dir and then re-create it, so in effect we | |
| 94 # remove all dirs and files under that directory. | |
| 95 shutil.rmtree(cls.tmp_dir) | |
| 96 os.mkdir(cls.tmp_dir) | |
| 97 | |
| 98 # Close test objects. | |
| 99 cls.resilient_close(cls.empty_src_key) | |
| 100 cls.resilient_close(cls.small_src_key) | |
| 101 cls.resilient_close(cls.larger_src_key) | |
| 102 | |
| 103 @classmethod | |
| 104 def build_test_input_object(cls, obj_name, size, debug): | |
| 105 buf = [] | |
| 106 for i in range(size): | |
| 107 buf.append(str(random.randint(0, 9))) | |
| 108 string_data = ''.join(buf) | |
| 109 uri = cls.src_bucket_uri.clone_replace_name(obj_name) | |
| 110 key = uri.new_key(validate=False) | |
| 111 key.set_contents_from_file(StringIO.StringIO(string_data)) | |
| 112 # Set debug on key's connection after creating data, so only the test | |
| 113 # runs will show HTTP output (if called passed debug>0). | |
| 114 key.bucket.connection.debug = debug | |
| 115 return (string_data, key) | |
| 116 | |
| 117 @classmethod | |
| 118 def set_up_class(cls, debug): | |
| 119 """ | |
| 120 Initializes test suite. | |
| 121 """ | |
| 122 | |
| 123 # Create the test bucket. | |
| 124 hostname = socket.gethostname().split('.')[0] | |
| 125 uri_base_str = 'gs://res_download_test_%s_%s_%s' % ( | |
| 126 hostname, os.getpid(), int(time.time())) | |
| 127 cls.src_bucket_uri = storage_uri('%s_dst' % uri_base_str) | |
| 128 cls.src_bucket_uri.create_bucket() | |
| 129 | |
| 130 # Create test source objects. | |
| 131 cls.empty_src_key_size = 0 | |
| 132 (cls.empty_src_key_as_string, cls.empty_src_key) = ( | |
| 133 cls.build_test_input_object('empty', cls.empty_src_key_size, | |
| 134 debug=debug)) | |
| 135 cls.small_src_key_size = 2 * 1024 # 2 KB. | |
| 136 (cls.small_src_key_as_string, cls.small_src_key) = ( | |
| 137 cls.build_test_input_object('small', cls.small_src_key_size, | |
| 138 debug=debug)) | |
| 139 cls.larger_src_key_size = 500 * 1024 # 500 KB. | |
| 140 (cls.larger_src_key_as_string, cls.larger_src_key) = ( | |
| 141 cls.build_test_input_object('larger', cls.larger_src_key_size, | |
| 142 debug=debug)) | |
| 143 | |
| 144 # Use a designated tmpdir prefix to make it easy to find the end of | |
| 145 # the tmp path. | |
| 146 cls.tmpdir_prefix = 'tmp_resumable_download_test' | |
| 147 | |
| 148 # Create temp dir and name for download file. | |
| 149 cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix) | |
| 150 cls.dst_file_name = '%s%sdst_file' % (cls.tmp_dir, os.sep) | |
| 151 | |
| 152 cls.tracker_file_name = '%s%stracker' % (cls.tmp_dir, os.sep) | |
| 153 | |
| 154 cls.created_test_data = True | |
| 155 | |
| 156 @classmethod | |
| 157 def tear_down_class(cls): | |
| 158 """ | |
| 159 Deletes test objects and bucket and tmp dir created by set_up_class. | |
| 160 """ | |
| 161 if not hasattr(cls, 'created_test_data'): | |
| 162 return | |
| 163 # Call cls.tearDown() in case the tests got interrupted, to ensure | |
| 164 # dst objects get deleted. | |
| 165 cls.tearDown() | |
| 166 | |
| 167 # Delete test objects. | |
| 168 cls.empty_src_key.delete() | |
| 169 cls.small_src_key.delete() | |
| 170 cls.larger_src_key.delete() | |
| 171 | |
| 172 # Retry (for up to 2 minutes) the bucket gets deleted (it may not | |
| 173 # the first time round, due to eventual consistency of bucket delete | |
| 174 # operations). | |
| 175 for i in range(60): | |
| 176 try: | |
| 177 cls.src_bucket_uri.delete_bucket() | |
| 178 break | |
| 179 except StorageResponseError: | |
| 180 print 'Test bucket (%s) not yet deleted, still trying' % ( | |
| 181 cls.src_bucket_uri.uri) | |
| 182 time.sleep(2) | |
| 183 shutil.rmtree(cls.tmp_dir) | |
| 184 cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix) | |
| 185 | |
| 186 def test_non_resumable_download(self): | |
| 187 """ | |
| 188 Tests that non-resumable downloads work | |
| 189 """ | |
| 190 self.small_src_key.get_contents_to_file(self.dst_fp) | |
| 191 self.assertEqual(self.small_src_key_size, | |
| 192 get_cur_file_size(self.dst_fp)) | |
| 193 self.assertEqual(self.small_src_key_as_string, | |
| 194 self.small_src_key.get_contents_as_string()) | |
| 195 | |
| 196 def test_download_without_persistent_tracker(self): | |
| 197 """ | |
| 198 Tests a single resumable download, with no tracker persistence | |
| 199 """ | |
| 200 res_download_handler = ResumableDownloadHandler() | |
| 201 self.small_src_key.get_contents_to_file( | |
| 202 self.dst_fp, res_download_handler=res_download_handler) | |
| 203 self.assertEqual(self.small_src_key_size, | |
| 204 get_cur_file_size(self.dst_fp)) | |
| 205 self.assertEqual(self.small_src_key_as_string, | |
| 206 self.small_src_key.get_contents_as_string()) | |
| 207 | |
| 208 def test_failed_download_with_persistent_tracker(self): | |
| 209 """ | |
| 210 Tests that failed resumable download leaves a correct tracker file | |
| 211 """ | |
| 212 harnass = CallbackTestHarnass() | |
| 213 res_download_handler = ResumableDownloadHandler( | |
| 214 tracker_file_name=self.tracker_file_name, num_retries=0) | |
| 215 try: | |
| 216 self.small_src_key.get_contents_to_file( | |
| 217 self.dst_fp, cb=harnass.call, | |
| 218 res_download_handler=res_download_handler) | |
| 219 self.fail('Did not get expected ResumableDownloadException') | |
| 220 except ResumableDownloadException, e: | |
| 221 # We'll get a ResumableDownloadException at this point because | |
| 222 # of CallbackTestHarnass (above). Check that the tracker file was | |
| 223 # created correctly. | |
| 224 self.assertEqual(e.disposition, | |
| 225 ResumableTransferDisposition.ABORT_CUR_PROCESS) | |
| 226 self.assertTrue(os.path.exists(self.tracker_file_name)) | |
| 227 f = open(self.tracker_file_name) | |
| 228 etag_line = f.readline() | |
| 229 m = re.search(ResumableDownloadHandler.ETAG_REGEX, etag_line) | |
| 230 f.close() | |
| 231 self.assertTrue(m) | |
| 232 | |
| 233 def test_retryable_exception_recovery(self): | |
| 234 """ | |
| 235 Tests handling of a retryable exception | |
| 236 """ | |
| 237 # Test one of the RETRYABLE_EXCEPTIONS. | |
| 238 exception = ResumableDownloadHandler.RETRYABLE_EXCEPTIONS[0] | |
| 239 harnass = CallbackTestHarnass(exception=exception) | |
| 240 res_download_handler = ResumableDownloadHandler(num_retries=1) | |
| 241 self.small_src_key.get_contents_to_file( | |
| 242 self.dst_fp, cb=harnass.call, | |
| 243 res_download_handler=res_download_handler) | |
| 244 # Ensure downloaded object has correct content. | |
| 245 self.assertEqual(self.small_src_key_size, | |
| 246 get_cur_file_size(self.dst_fp)) | |
| 247 self.assertEqual(self.small_src_key_as_string, | |
| 248 self.small_src_key.get_contents_as_string()) | |
| 249 | |
| 250 def test_broken_pipe_recovery(self): | |
| 251 """ | |
| 252 Tests handling of a Broken Pipe (which interacts with an httplib bug) | |
| 253 """ | |
| 254 exception = IOError(errno.EPIPE, "Broken pipe") | |
| 255 harnass = CallbackTestHarnass(exception=exception) | |
| 256 res_download_handler = ResumableDownloadHandler(num_retries=1) | |
| 257 self.small_src_key.get_contents_to_file( | |
| 258 self.dst_fp, cb=harnass.call, | |
| 259 res_download_handler=res_download_handler) | |
| 260 # Ensure downloaded object has correct content. | |
| 261 self.assertEqual(self.small_src_key_size, | |
| 262 get_cur_file_size(self.dst_fp)) | |
| 263 self.assertEqual(self.small_src_key_as_string, | |
| 264 self.small_src_key.get_contents_as_string()) | |
| 265 | |
| 266 def test_non_retryable_exception_handling(self): | |
| 267 """ | |
| 268 Tests resumable download that fails with a non-retryable exception | |
| 269 """ | |
| 270 harnass = CallbackTestHarnass( | |
| 271 exception=OSError(errno.EACCES, 'Permission denied')) | |
| 272 res_download_handler = ResumableDownloadHandler(num_retries=1) | |
| 273 try: | |
| 274 self.small_src_key.get_contents_to_file( | |
| 275 self.dst_fp, cb=harnass.call, | |
| 276 res_download_handler=res_download_handler) | |
| 277 self.fail('Did not get expected OSError') | |
| 278 except OSError, e: | |
| 279 # Ensure the error was re-raised. | |
| 280 self.assertEqual(e.errno, 13) | |
| 281 | |
| 282 def test_failed_and_restarted_download_with_persistent_tracker(self): | |
| 283 """ | |
| 284 Tests resumable download that fails once and then completes, | |
| 285 with tracker file | |
| 286 """ | |
| 287 harnass = CallbackTestHarnass() | |
| 288 res_download_handler = ResumableDownloadHandler( | |
| 289 tracker_file_name=self.tracker_file_name, num_retries=1) | |
| 290 self.small_src_key.get_contents_to_file( | |
| 291 self.dst_fp, cb=harnass.call, | |
| 292 res_download_handler=res_download_handler) | |
| 293 # Ensure downloaded object has correct content. | |
| 294 self.assertEqual(self.small_src_key_size, | |
| 295 get_cur_file_size(self.dst_fp)) | |
| 296 self.assertEqual(self.small_src_key_as_string, | |
| 297 self.small_src_key.get_contents_as_string()) | |
| 298 # Ensure tracker file deleted. | |
| 299 self.assertFalse(os.path.exists(self.tracker_file_name)) | |
| 300 | |
| 301 def test_multiple_in_process_failures_then_succeed(self): | |
| 302 """ | |
| 303 Tests resumable download that fails twice in one process, then completes | |
| 304 """ | |
| 305 res_download_handler = ResumableDownloadHandler(num_retries=3) | |
| 306 self.small_src_key.get_contents_to_file( | |
| 307 self.dst_fp, res_download_handler=res_download_handler) | |
| 308 # Ensure downloaded object has correct content. | |
| 309 self.assertEqual(self.small_src_key_size, | |
| 310 get_cur_file_size(self.dst_fp)) | |
| 311 self.assertEqual(self.small_src_key_as_string, | |
| 312 self.small_src_key.get_contents_as_string()) | |
| 313 | |
| 314 def test_multiple_in_process_failures_then_succeed_with_tracker_file(self): | |
| 315 """ | |
| 316 Tests resumable download that fails completely in one process, | |
| 317 then when restarted completes, using a tracker file | |
| 318 """ | |
| 319 # Set up test harnass that causes more failures than a single | |
| 320 # ResumableDownloadHandler instance will handle, writing enough data | |
| 321 # before the first failure that some of it survives that process run. | |
| 322 harnass = CallbackTestHarnass( | |
| 323 fail_after_n_bytes=self.larger_src_key_size/2, num_times_to_fail=2) | |
| 324 res_download_handler = ResumableDownloadHandler( | |
| 325 tracker_file_name=self.tracker_file_name, num_retries=0) | |
| 326 try: | |
| 327 self.larger_src_key.get_contents_to_file( | |
| 328 self.dst_fp, cb=harnass.call, | |
| 329 res_download_handler=res_download_handler) | |
| 330 self.fail('Did not get expected ResumableDownloadException') | |
| 331 except ResumableDownloadException, e: | |
| 332 self.assertEqual(e.disposition, | |
| 333 ResumableTransferDisposition.ABORT_CUR_PROCESS) | |
| 334 # Ensure a tracker file survived. | |
| 335 self.assertTrue(os.path.exists(self.tracker_file_name)) | |
| 336 # Try it one more time; this time should succeed. | |
| 337 self.larger_src_key.get_contents_to_file( | |
| 338 self.dst_fp, cb=harnass.call, | |
| 339 res_download_handler=res_download_handler) | |
| 340 self.assertEqual(self.larger_src_key_size, | |
| 341 get_cur_file_size(self.dst_fp)) | |
| 342 self.assertEqual(self.larger_src_key_as_string, | |
| 343 self.larger_src_key.get_contents_as_string()) | |
| 344 self.assertFalse(os.path.exists(self.tracker_file_name)) | |
| 345 # Ensure some of the file was downloaded both before and after failure. | |
| 346 self.assertTrue( | |
| 347 len(harnass.transferred_seq_before_first_failure) > 1 and | |
| 348 len(harnass.transferred_seq_after_first_failure) > 1) | |
| 349 | |
| 350 def test_download_with_inital_partial_download_before_failure(self): | |
| 351 """ | |
| 352 Tests resumable download that successfully downloads some content | |
| 353 before it fails, then restarts and completes | |
| 354 """ | |
| 355 # Set up harnass to fail download after several hundred KB so download | |
| 356 # server will have saved something before we retry. | |
| 357 harnass = CallbackTestHarnass( | |
| 358 fail_after_n_bytes=self.larger_src_key_size/2) | |
| 359 res_download_handler = ResumableDownloadHandler(num_retries=1) | |
| 360 self.larger_src_key.get_contents_to_file( | |
| 361 self.dst_fp, cb=harnass.call, | |
| 362 res_download_handler=res_download_handler) | |
| 363 # Ensure downloaded object has correct content. | |
| 364 self.assertEqual(self.larger_src_key_size, | |
| 365 get_cur_file_size(self.dst_fp)) | |
| 366 self.assertEqual(self.larger_src_key_as_string, | |
| 367 self.larger_src_key.get_contents_as_string()) | |
| 368 # Ensure some of the file was downloaded both before and after failure. | |
| 369 self.assertTrue( | |
| 370 len(harnass.transferred_seq_before_first_failure) > 1 and | |
| 371 len(harnass.transferred_seq_after_first_failure) > 1) | |
| 372 | |
| 373 def test_zero_length_object_download(self): | |
| 374 """ | |
| 375 Tests downloading a zero-length object (exercises boundary conditions). | |
| 376 """ | |
| 377 res_download_handler = ResumableDownloadHandler() | |
| 378 self.empty_src_key.get_contents_to_file( | |
| 379 self.dst_fp, res_download_handler=res_download_handler) | |
| 380 self.assertEqual(0, get_cur_file_size(self.dst_fp)) | |
| 381 | |
| 382 def test_download_with_object_size_change_between_starts(self): | |
| 383 """ | |
| 384 Tests resumable download on an object that changes sizes between inital | |
| 385 download start and restart | |
| 386 """ | |
| 387 harnass = CallbackTestHarnass( | |
| 388 fail_after_n_bytes=self.larger_src_key_size/2, num_times_to_fail=2) | |
| 389 # Set up first process' ResumableDownloadHandler not to do any | |
| 390 # retries (initial download request will establish expected size to | |
| 391 # download server). | |
| 392 res_download_handler = ResumableDownloadHandler( | |
| 393 tracker_file_name=self.tracker_file_name, num_retries=0) | |
| 394 try: | |
| 395 self.larger_src_key.get_contents_to_file( | |
| 396 self.dst_fp, cb=harnass.call, | |
| 397 res_download_handler=res_download_handler) | |
| 398 self.fail('Did not get expected ResumableDownloadException') | |
| 399 except ResumableDownloadException, e: | |
| 400 # First abort (from harnass-forced failure) should be | |
| 401 # ABORT_CUR_PROCESS. | |
| 402 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_C
UR_PROCESS) | |
| 403 # Ensure a tracker file survived. | |
| 404 self.assertTrue(os.path.exists(self.tracker_file_name)) | |
| 405 # Try it again, this time with different src key (simulating an | |
| 406 # object that changes sizes between downloads). | |
| 407 try: | |
| 408 self.small_src_key.get_contents_to_file( | |
| 409 self.dst_fp, res_download_handler=res_download_handler) | |
| 410 self.fail('Did not get expected ResumableDownloadException') | |
| 411 except ResumableDownloadException, e: | |
| 412 # This abort should be a hard abort (object size changing during | |
| 413 # transfer). | |
| 414 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 415 self.assertNotEqual( | |
| 416 e.message.find('md5 signature doesn\'t match etag'), -1) | |
| 417 | |
| 418 def test_download_with_file_content_change_during_download(self): | |
| 419 """ | |
| 420 Tests resumable download on an object where the file content changes | |
| 421 without changing length while download in progress | |
| 422 """ | |
| 423 harnass = CallbackTestHarnass( | |
| 424 fail_after_n_bytes=self.larger_src_key_size/2, num_times_to_fail=2) | |
| 425 # Set up first process' ResumableDownloadHandler not to do any | |
| 426 # retries (initial download request will establish expected size to | |
| 427 # download server). | |
| 428 res_download_handler = ResumableDownloadHandler( | |
| 429 tracker_file_name=self.tracker_file_name, num_retries=0) | |
| 430 dst_filename = self.dst_fp.name | |
| 431 try: | |
| 432 self.larger_src_key.get_contents_to_file( | |
| 433 self.dst_fp, cb=harnass.call, | |
| 434 res_download_handler=res_download_handler) | |
| 435 self.fail('Did not get expected ResumableDownloadException') | |
| 436 except ResumableDownloadException, e: | |
| 437 # First abort (from harnass-forced failure) should be | |
| 438 # ABORT_CUR_PROCESS. | |
| 439 self.assertEqual(e.disposition, | |
| 440 ResumableTransferDisposition.ABORT_CUR_PROCESS) | |
| 441 # Ensure a tracker file survived. | |
| 442 self.assertTrue(os.path.exists(self.tracker_file_name)) | |
| 443 # Before trying again change the first byte of the file fragment | |
| 444 # that was already downloaded. | |
| 445 orig_size = get_cur_file_size(self.dst_fp) | |
| 446 self.dst_fp.seek(0, os.SEEK_SET) | |
| 447 self.dst_fp.write('a') | |
| 448 # Ensure the file size didn't change. | |
| 449 self.assertEqual(orig_size, get_cur_file_size(self.dst_fp)) | |
| 450 try: | |
| 451 self.larger_src_key.get_contents_to_file( | |
| 452 self.dst_fp, cb=harnass.call, | |
| 453 res_download_handler=res_download_handler) | |
| 454 self.fail('Did not get expected ResumableDownloadException') | |
| 455 except ResumableDownloadException, e: | |
| 456 # This abort should be a hard abort (file content changing during | |
| 457 # transfer). | |
| 458 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 459 self.assertNotEqual( | |
| 460 e.message.find('md5 signature doesn\'t match etag'), -1) | |
| 461 # Ensure the bad data wasn't left around. | |
| 462 self.assertFalse(os.path.exists(dst_filename)) | |
| 463 | |
| 464 def test_download_with_invalid_tracker_etag(self): | |
| 465 """ | |
| 466 Tests resumable download with a tracker file containing an invalid etag | |
| 467 """ | |
| 468 invalid_etag_tracker_file_name = ( | |
| 469 '%s%sinvalid_etag_tracker' % (self.tmp_dir, os.sep)) | |
| 470 f = open(invalid_etag_tracker_file_name, 'w') | |
| 471 f.write('3.14159\n') | |
| 472 f.close() | |
| 473 res_download_handler = ResumableDownloadHandler( | |
| 474 tracker_file_name=invalid_etag_tracker_file_name) | |
| 475 # An error should be printed about the invalid tracker, but then it | |
| 476 # should run the update successfully. | |
| 477 self.small_src_key.get_contents_to_file( | |
| 478 self.dst_fp, res_download_handler=res_download_handler) | |
| 479 self.assertEqual(self.small_src_key_size, | |
| 480 get_cur_file_size(self.dst_fp)) | |
| 481 self.assertEqual(self.small_src_key_as_string, | |
| 482 self.small_src_key.get_contents_as_string()) | |
| 483 | |
| 484 def test_download_with_inconsistent_etag_in_tracker(self): | |
| 485 """ | |
| 486 Tests resumable download with an inconsistent etag in tracker file | |
| 487 """ | |
| 488 inconsistent_etag_tracker_file_name = ( | |
| 489 '%s%sinconsistent_etag_tracker' % (self.tmp_dir, os.sep)) | |
| 490 f = open(inconsistent_etag_tracker_file_name, 'w') | |
| 491 good_etag = self.small_src_key.etag.strip('"\'') | |
| 492 new_val_as_list = [] | |
| 493 for c in reversed(good_etag): | |
| 494 new_val_as_list.append(c) | |
| 495 f.write('%s\n' % ''.join(new_val_as_list)) | |
| 496 f.close() | |
| 497 res_download_handler = ResumableDownloadHandler( | |
| 498 tracker_file_name=inconsistent_etag_tracker_file_name) | |
| 499 # An error should be printed about the expired tracker, but then it | |
| 500 # should run the update successfully. | |
| 501 self.small_src_key.get_contents_to_file( | |
| 502 self.dst_fp, res_download_handler=res_download_handler) | |
| 503 self.assertEqual(self.small_src_key_size, | |
| 504 get_cur_file_size(self.dst_fp)) | |
| 505 self.assertEqual(self.small_src_key_as_string, | |
| 506 self.small_src_key.get_contents_as_string()) | |
| 507 | |
| 508 def test_download_with_unwritable_tracker_file(self): | |
| 509 """ | |
| 510 Tests resumable download with an unwritable tracker file | |
| 511 """ | |
| 512 # Make dir where tracker_file lives temporarily unwritable. | |
| 513 save_mod = os.stat(self.tmp_dir).st_mode | |
| 514 try: | |
| 515 os.chmod(self.tmp_dir, 0) | |
| 516 res_download_handler = ResumableDownloadHandler( | |
| 517 tracker_file_name=self.tracker_file_name) | |
| 518 except ResumableDownloadException, e: | |
| 519 self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) | |
| 520 self.assertNotEqual( | |
| 521 e.message.find('Couldn\'t write URI tracker file'), -1) | |
| 522 finally: | |
| 523 # Restore original protection of dir where tracker_file lives. | |
| 524 os.chmod(self.tmp_dir, save_mod) | |
| 525 | |
| 526 if __name__ == '__main__': | |
| 527 if sys.version_info[:3] < (2, 5, 1): | |
| 528 sys.exit('These tests must be run on at least Python 2.5.1\n') | |
| 529 | |
| 530 # Use -d to see more HTTP protocol detail during tests. Note that | |
| 531 # unlike the upload test case, you won't see much for the downloads | |
| 532 # because there's no HTTP server state protocol for in the download case | |
| 533 # (and the actual Range GET HTTP protocol detail is suppressed by the | |
| 534 # normal boto.s3.Key.get_file() processing). | |
| 535 debug = 0 | |
| 536 opts, args = getopt.getopt(sys.argv[1:], 'd', ['debug']) | |
| 537 for o, a in opts: | |
| 538 if o in ('-d', '--debug'): | |
| 539 debug = 2 | |
| 540 | |
| 541 test_loader = unittest.TestLoader() | |
| 542 test_loader.testMethodPrefix = 'test_' | |
| 543 suite = test_loader.loadTestsFromTestCase(ResumableDownloadTests) | |
| 544 # Seems like there should be a cleaner way to find the test_class. | |
| 545 test_class = suite.__getattribute__('_tests')[0] | |
| 546 # We call set_up_class() and tear_down_class() ourselves because we | |
| 547 # don't assume the user has Python 2.7 (which supports classmethods | |
| 548 # that do it, with camelCase versions of these names). | |
| 549 try: | |
| 550 print 'Setting up %s...' % test_class.get_suite_description() | |
| 551 test_class.set_up_class(debug) | |
| 552 print 'Running %s...' % test_class.get_suite_description() | |
| 553 unittest.TextTestRunner(verbosity=2).run(suite) | |
| 554 finally: | |
| 555 print 'Cleaning up after %s...' % test_class.get_suite_description() | |
| 556 test_class.tear_down_class() | |
| 557 print '' | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 10199002:
Upgrade gsutil to 3.4 (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart