OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Download files from Google Storage based on SHA1 sums.""" | 6 """Download files from Google Storage based on SHA1 sums.""" |
7 | 7 |
8 | 8 |
9 import hashlib | 9 import hashlib |
10 import optparse | 10 import optparse |
(...skipping 17 matching lines...) Expand all Loading... |
28 | 28 |
29 | 29 |
30 class InvalidFileError(IOError): | 30 class InvalidFileError(IOError): |
31 pass | 31 pass |
32 | 32 |
33 | 33 |
34 # Common utilities | 34 # Common utilities |
35 class Gsutil(object): | 35 class Gsutil(object): |
36 """Call gsutil with some predefined settings. This is a convenience object, | 36 """Call gsutil with some predefined settings. This is a convenience object, |
37 and is also immutable.""" | 37 and is also immutable.""" |
38 def __init__(self, path, boto_path=None, timeout=None): | 38 def __init__(self, path, boto_path, timeout=None): |
39 if not os.path.exists(path): | 39 if not os.path.exists(path): |
40 raise FileNotFoundError('GSUtil not found in %s' % path) | 40 raise FileNotFoundError('GSUtil not found in %s' % path) |
41 self.path = path | 41 self.path = path |
42 self.timeout = timeout | 42 self.timeout = timeout |
43 self.boto_path = boto_path | 43 self.boto_path = boto_path |
44 | 44 |
45 def call(self, *args): | 45 def call(self, *args): |
46 env = os.environ.copy() | 46 env = os.environ.copy() |
47 if self.boto_path: | 47 if self.boto_path: |
48 env['AWS_CREDENTIAL_FILE'] = self.boto_path | 48 env['AWS_CREDENTIAL_FILE'] = self.boto_path |
(...skipping 257 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
306 parser.error('Unreachable state.') | 306 parser.error('Unreachable state.') |
307 | 307 |
308 # Check if output file already exists. | 308 # Check if output file already exists. |
309 if not options.directory and not options.force and not options.no_resume: | 309 if not options.directory and not options.force and not options.no_resume: |
310 if os.path.exists(options.output): | 310 if os.path.exists(options.output): |
311 parser.error('Output file %s exists and --no_resume is specified.' | 311 parser.error('Output file %s exists and --no_resume is specified.' |
312 % options.output) | 312 % options.output) |
313 | 313 |
314 # Make sure we can find a working instance of gsutil. | 314 # Make sure we can find a working instance of gsutil. |
315 if os.path.exists(GSUTIL_DEFAULT_PATH): | 315 if os.path.exists(GSUTIL_DEFAULT_PATH): |
316 gsutil = Gsutil(GSUTIL_DEFAULT_PATH) | 316 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
317 else: | 317 else: |
318 gsutil = None | 318 gsutil = None |
319 for path in os.environ["PATH"].split(os.pathsep): | 319 for path in os.environ["PATH"].split(os.pathsep): |
320 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 320 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
321 gsutil = Gsutil(os.path.join(path, 'gsutil')) | 321 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
322 if not gsutil: | 322 if not gsutil: |
323 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 323 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
324 GSUTIL_DEFAULT_PATH) | 324 GSUTIL_DEFAULT_PATH) |
325 | 325 |
326 # Check we have a valid bucket with valid permissions. | 326 # Check we have a valid bucket with valid permissions. |
327 base_url, code = check_bucket_permissions(options.bucket, gsutil) | 327 base_url, code = check_bucket_permissions(options.bucket, gsutil) |
328 if code: | 328 if code: |
329 return code | 329 return code |
330 | 330 |
331 return download_from_google_storage( | 331 return download_from_google_storage( |
332 input_filename, base_url, gsutil, options.num_threads, options.directory, | 332 input_filename, base_url, gsutil, options.num_threads, options.directory, |
333 options.recursive, options.force, options.output, options.ignore_errors, | 333 options.recursive, options.force, options.output, options.ignore_errors, |
334 options.sha1_file) | 334 options.sha1_file) |
335 | 335 |
336 | 336 |
337 if __name__ == '__main__': | 337 if __name__ == '__main__': |
338 sys.exit(main(sys.argv)) | 338 sys.exit(main(sys.argv)) |
OLD | NEW |