OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Download files from Google Storage based on SHA1 sums.""" | 6 """Download files from Google Storage based on SHA1 sums.""" |
7 | 7 |
8 | 8 |
9 import hashlib | 9 import hashlib |
10 import optparse | 10 import optparse |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
59 if not os.path.exists(path): | 59 if not os.path.exists(path): |
60 raise FileNotFoundError('GSUtil not found in %s' % path) | 60 raise FileNotFoundError('GSUtil not found in %s' % path) |
61 self.path = path | 61 self.path = path |
62 self.timeout = timeout | 62 self.timeout = timeout |
63 self.boto_path = boto_path | 63 self.boto_path = boto_path |
64 self.bypass_prodaccess = bypass_prodaccess | 64 self.bypass_prodaccess = bypass_prodaccess |
65 | 65 |
66 def get_sub_env(self): | 66 def get_sub_env(self): |
67 env = os.environ.copy() | 67 env = os.environ.copy() |
68 if self.boto_path == os.devnull: | 68 if self.boto_path == os.devnull: |
69 env['AWS_CREDENTIAL_FILE'] = '' | 69 env.pop('AWS_CREDENTIAL_FILE', None) |
70 env['BOTO_CONFIG'] = '' | 70 env['BOTO_CONFIG'] = '' |
71 elif self.boto_path: | 71 elif self.boto_path: |
72 env['AWS_CREDENTIAL_FILE'] = self.boto_path | 72 env['AWS_CREDENTIAL_FILE'] = self.boto_path |
73 env['BOTO_CONFIG'] = self.boto_path | 73 env['BOTO_CONFIG'] = self.boto_path |
74 else: | 74 else: |
75 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools' | 75 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools' |
76 custompath = os.path.expanduser(custompath) | 76 custompath = os.path.expanduser(custompath) |
77 if os.path.exists(custompath): | 77 if os.path.exists(custompath): |
78 env['AWS_CREDENTIAL_FILE'] = custompath | 78 env['AWS_CREDENTIAL_FILE'] = custompath |
79 | 79 |
80 return env | 80 return env |
81 | 81 |
82 def call(self, *args): | 82 def call(self, *args): |
83 cmd = [sys.executable, self.path] | 83 cmd = [sys.executable, self.path] |
84 if self.bypass_prodaccess: | 84 if self.bypass_prodaccess: |
85 cmd.append('--bypass_prodaccess') | 85 cmd.append('--bypass_prodaccess') |
86 cmd.extend(args) | 86 cmd.extend(args) |
87 return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout) | 87 return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout) |
88 | 88 |
89 def check_call(self, *args): | 89 def check_call(self, *args, **kwargs): |
| 90 def tee(stream, buf): |
| 91 def _inner(char): |
| 92 stream.write(char) |
| 93 buf.append(char) |
| 94 return _inner |
| 95 |
90 cmd = [sys.executable, self.path] | 96 cmd = [sys.executable, self.path] |
91 if self.bypass_prodaccess: | 97 if self.bypass_prodaccess: |
92 cmd.append('--bypass_prodaccess') | 98 cmd.append('--bypass_prodaccess') |
93 cmd.extend(args) | 99 cmd.extend(args) |
94 ((out, err), code) = subprocess2.communicate( | 100 |
95 cmd, | 101 out = [] |
96 stdout=subprocess2.PIPE, | 102 err = [] |
97 stderr=subprocess2.PIPE, | 103 |
98 env=self.get_sub_env(), | 104 proc = subprocess2.Popen(cmd, env=self.get_sub_env(), |
99 timeout=self.timeout) | 105 stdout=subprocess2.PIPE, |
| 106 stderr=subprocess2.PIPE) |
| 107 if kwargs.get('verbose'): |
| 108 proc.stdout_cb = tee(sys.stdout, out) |
| 109 proc.stderr_cb = tee(sys.stderr, err) |
| 110 else: |
| 111 proc.stdout_cb = out.append |
| 112 proc.stderr_cb = err.append |
| 113 proc.communicate(timeout=self.timeout) |
| 114 code = proc.returncode |
| 115 |
| 116 out = ''.join(out) |
| 117 err = ''.join(err) |
100 | 118 |
101 # Parse output. | 119 # Parse output. |
102 status_code_match = re.search('status=([0-9]+)', err) | 120 status_code_match = re.search('status=([0-9]+)', err) |
103 if status_code_match: | 121 if status_code_match: |
104 return (int(status_code_match.group(1)), out, err) | 122 return (int(status_code_match.group(1)), out, err) |
105 if ('You are attempting to access protected data with ' | 123 if ('You are attempting to access protected data with ' |
106 'no configured credentials.' in err): | 124 'no configured credentials.' in err): |
107 return (403, out, err) | 125 return (403, out, err) |
108 if 'No such object' in err: | 126 if 'No such object' in err: |
109 return (404, out, err) | 127 return (404, out, err) |
(...skipping 348 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
458 return code | 476 return code |
459 | 477 |
460 return download_from_google_storage( | 478 return download_from_google_storage( |
461 input_filename, base_url, gsutil, options.num_threads, options.directory, | 479 input_filename, base_url, gsutil, options.num_threads, options.directory, |
462 options.recursive, options.force, options.output, options.ignore_errors, | 480 options.recursive, options.force, options.output, options.ignore_errors, |
463 options.sha1_file, options.verbose, options.auto_platform) | 481 options.sha1_file, options.verbose, options.auto_platform) |
464 | 482 |
465 | 483 |
466 if __name__ == '__main__': | 484 if __name__ == '__main__': |
467 sys.exit(main(sys.argv)) | 485 sys.exit(main(sys.argv)) |
OLD | NEW |