Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Download files from Google Storage based on SHA1 sums.""" | 6 """Download files from Google Storage based on SHA1 sums.""" |
| 7 | 7 |
| 8 | 8 |
| 9 import hashlib | 9 import hashlib |
| 10 import optparse | 10 import optparse |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 79 return (code, out, err) | 79 return (code, out, err) |
| 80 | 80 |
| 81 | 81 |
| 82 def check_bucket_permissions(bucket, gsutil): | 82 def check_bucket_permissions(bucket, gsutil): |
| 83 if not bucket: | 83 if not bucket: |
| 84 print >> sys.stderr, 'Missing bucket %s.' | 84 print >> sys.stderr, 'Missing bucket %s.' |
| 85 return (None, 1) | 85 return (None, 1) |
| 86 base_url = 'gs://%s' % bucket | 86 base_url = 'gs://%s' % bucket |
| 87 | 87 |
| 88 code, _, ls_err = gsutil.check_call('ls', base_url) | 88 code, _, ls_err = gsutil.check_call('ls', base_url) |
| 89 if code != 0: | |
| 90 print >> sys.stderr, ls_err | |
|
Isaac (away)
2013/09/19 17:57:59
Optional nit: consider wrapping this so it's clear
Ryan Tseng
2013/09/19 18:05:51
Yeah I think gsutil already does a pretty good job
| |
| 89 if code == 403: | 91 if code == 403: |
| 90 print >> sys.stderr, 'Got error 403 while authenticating to %s.' % base_url | 92 print >> sys.stderr, 'Got error 403 while authenticating to %s.' % base_url |
| 91 print >> sys.stderr, 'Try running "download_from_google_storage --config".' | 93 print >> sys.stderr, 'Try running "download_from_google_storage --config".' |
| 92 elif code == 404: | 94 elif code == 404: |
| 93 print >> sys.stderr, '%s not found.' % base_url | 95 print >> sys.stderr, '%s not found.' % base_url |
| 94 elif code != 0: | |
| 95 print >> sys.stderr, ls_err | |
| 96 return (base_url, code) | 96 return (base_url, code) |
| 97 | 97 |
| 98 | 98 |
| 99 def get_sha1(filename): | 99 def get_sha1(filename): |
| 100 sha1 = hashlib.sha1() | 100 sha1 = hashlib.sha1() |
| 101 with open(filename, 'rb') as f: | 101 with open(filename, 'rb') as f: |
| 102 while True: | 102 while True: |
| 103 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. | 103 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. |
| 104 chunk = f.read(1024*1024) | 104 chunk = f.read(1024*1024) |
| 105 if not chunk: | 105 if not chunk: |
| (...skipping 237 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 343 return code | 343 return code |
| 344 | 344 |
| 345 return download_from_google_storage( | 345 return download_from_google_storage( |
| 346 input_filename, base_url, gsutil, options.num_threads, options.directory, | 346 input_filename, base_url, gsutil, options.num_threads, options.directory, |
| 347 options.recursive, options.force, options.output, options.ignore_errors, | 347 options.recursive, options.force, options.output, options.ignore_errors, |
| 348 options.sha1_file) | 348 options.sha1_file) |
| 349 | 349 |
| 350 | 350 |
| 351 if __name__ == '__main__': | 351 if __name__ == '__main__': |
| 352 sys.exit(main(sys.argv)) | 352 sys.exit(main(sys.argv)) |
| OLD | NEW |