OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Download files from Google Storage based on SHA1 sums.""" | 6 """Download files from Google Storage based on SHA1 sums.""" |
7 | 7 |
8 | 8 |
9 import hashlib | 9 import hashlib |
10 import optparse | 10 import optparse |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
75 | 75 |
76 def check_bucket_permissions(bucket, gsutil): | 76 def check_bucket_permissions(bucket, gsutil): |
77 if not bucket: | 77 if not bucket: |
78 print >> sys.stderr, 'Missing bucket %s.' | 78 print >> sys.stderr, 'Missing bucket %s.' |
79 return (None, 1) | 79 return (None, 1) |
80 base_url = 'gs://%s' % bucket | 80 base_url = 'gs://%s' % bucket |
81 | 81 |
82 code, _, ls_err = gsutil.check_call('ls', base_url) | 82 code, _, ls_err = gsutil.check_call('ls', base_url) |
83 if code == 403: | 83 if code == 403: |
84 print >> sys.stderr, 'Got error 403 while authenticating to %s.' % base_url | 84 print >> sys.stderr, 'Got error 403 while authenticating to %s.' % base_url |
85 print >> sys.stderr, 'Try running "gsutil config".' | 85 print >> sys.stderr, 'Try running "download_from_google_storage --config".' |
86 elif code == 404: | 86 elif code == 404: |
87 print >> sys.stderr, '%s not found.' % base_url | 87 print >> sys.stderr, '%s not found.' % base_url |
88 elif code != 0: | 88 elif code != 0: |
89 print >> sys.stderr, ls_err | 89 print >> sys.stderr, ls_err |
90 return (base_url, code) | 90 return (base_url, code) |
91 | 91 |
92 | 92 |
93 def get_sha1(filename): | 93 def get_sha1(filename): |
94 sha1 = hashlib.sha1() | 94 sha1 = hashlib.sha1() |
95 with open(filename, 'rb') as f: | 95 with open(filename, 'rb') as f: |
(...skipping 169 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
265 help='Scan folders recursively for .sha1 files. ' | 265 help='Scan folders recursively for .sha1 files. ' |
266 'Must be used with -d/--directory') | 266 'Must be used with -d/--directory') |
267 parser.add_option('-t', '--num_threads', default=1, type='int', | 267 parser.add_option('-t', '--num_threads', default=1, type='int', |
268 help='Number of downloader threads to run.') | 268 help='Number of downloader threads to run.') |
269 parser.add_option('-d', '--directory', action='store_true', | 269 parser.add_option('-d', '--directory', action='store_true', |
270 help='The target is a directory. ' | 270 help='The target is a directory. ' |
271 'Cannot be used with -s/--sha1_file.') | 271 'Cannot be used with -s/--sha1_file.') |
272 parser.add_option('-s', '--sha1_file', action='store_true', | 272 parser.add_option('-s', '--sha1_file', action='store_true', |
273 help='The target is a file containing a sha1 sum. ' | 273 help='The target is a file containing a sha1 sum. ' |
274 'Cannot be used with -d/--directory.') | 274 'Cannot be used with -d/--directory.') |
275 parser.add_option('-g', '--config', action='store_true', | |
276 help='Alias for "gsutil config". Run this if you want ' | |
277 'to initialize your saved Google Storage ' | |
278 'credentials.') | |
275 | 279 |
276 (options, args) = parser.parse_args() | 280 (options, args) = parser.parse_args() |
281 # First, make sure we can find a working instance of gsutil. | |
282 if os.path.exists(GSUTIL_DEFAULT_PATH): | |
283 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) | |
284 else: | |
285 gsutil = None | |
286 for path in os.environ["PATH"].split(os.pathsep): | |
287 if os.path.exists(path) and 'gsutil' in os.listdir(path): | |
288 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | |
M-A Ruel
2013/07/09 15:20:19
Add a "break" statement after, no need to continue
| |
289 if not gsutil: | |
290 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | |
291 GSUTIL_DEFAULT_PATH) | |
292 | |
293 # Passing in -g/--config will run our copy of GSUtil, then quit. | |
294 if options.config: | |
295 return gsutil.call('config') | |
296 | |
277 if not args: | 297 if not args: |
278 parser.error('Missing target.') | 298 parser.error('Missing target.') |
279 if len(args) > 1: | 299 if len(args) > 1: |
280 parser.error('Too many targets.') | 300 parser.error('Too many targets.') |
281 if not options.bucket: | 301 if not options.bucket: |
282 parser.error('Missing bucket. Specify bucket with --bucket.') | 302 parser.error('Missing bucket. Specify bucket with --bucket.') |
283 if options.sha1_file and options.directory: | 303 if options.sha1_file and options.directory: |
284 parser.error('Both --directory and --sha1_file are specified, ' | 304 parser.error('Both --directory and --sha1_file are specified, ' |
285 'can only specify one.') | 305 'can only specify one.') |
286 if options.recursive and not options.directory: | 306 if options.recursive and not options.directory: |
(...skipping 17 matching lines...) Expand all Loading... | |
304 options.output = input_filename[:-5] | 324 options.output = input_filename[:-5] |
305 else: | 325 else: |
306 parser.error('Unreachable state.') | 326 parser.error('Unreachable state.') |
307 | 327 |
308 # Check if output file already exists. | 328 # Check if output file already exists. |
309 if not options.directory and not options.force and not options.no_resume: | 329 if not options.directory and not options.force and not options.no_resume: |
310 if os.path.exists(options.output): | 330 if os.path.exists(options.output): |
311 parser.error('Output file %s exists and --no_resume is specified.' | 331 parser.error('Output file %s exists and --no_resume is specified.' |
312 % options.output) | 332 % options.output) |
313 | 333 |
314 # Make sure we can find a working instance of gsutil. | |
315 if os.path.exists(GSUTIL_DEFAULT_PATH): | |
316 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) | |
317 else: | |
318 gsutil = None | |
319 for path in os.environ["PATH"].split(os.pathsep): | |
320 if os.path.exists(path) and 'gsutil' in os.listdir(path): | |
321 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | |
322 if not gsutil: | |
323 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | |
324 GSUTIL_DEFAULT_PATH) | |
325 | |
326 # Check we have a valid bucket with valid permissions. | 334 # Check we have a valid bucket with valid permissions. |
327 base_url, code = check_bucket_permissions(options.bucket, gsutil) | 335 base_url, code = check_bucket_permissions(options.bucket, gsutil) |
328 if code: | 336 if code: |
329 return code | 337 return code |
330 | 338 |
331 return download_from_google_storage( | 339 return download_from_google_storage( |
332 input_filename, base_url, gsutil, options.num_threads, options.directory, | 340 input_filename, base_url, gsutil, options.num_threads, options.directory, |
333 options.recursive, options.force, options.output, options.ignore_errors, | 341 options.recursive, options.force, options.output, options.ignore_errors, |
334 options.sha1_file) | 342 options.sha1_file) |
335 | 343 |
336 | 344 |
337 if __name__ == '__main__': | 345 if __name__ == '__main__': |
338 sys.exit(main(sys.argv)) | 346 sys.exit(main(sys.argv)) |
OLD | NEW |