Index: download_from_google_storage.py |
diff --git a/download_from_google_storage.py b/download_from_google_storage.py |
index 8370515e6a15879c3ddf5c1e8693efeb595a8ccd..c184b87e8d5cb23c3c24b94cff04327505e12668 100755 |
--- a/download_from_google_storage.py |
+++ b/download_from_google_storage.py |
@@ -66,7 +66,7 @@ class Gsutil(object): |
def get_sub_env(self): |
env = os.environ.copy() |
if self.boto_path == os.devnull: |
- env['AWS_CREDENTIAL_FILE'] = '' |
+ env.pop('AWS_CREDENTIAL_FILE', None) |
Ryan Tseng
2014/04/18 00:28:53
???
iannucci
2014/04/28 21:05:28
Otherwise gsutil freaks out because the env var is
|
env['BOTO_CONFIG'] = '' |
elif self.boto_path: |
env['AWS_CREDENTIAL_FILE'] = self.boto_path |
@@ -86,17 +86,35 @@ class Gsutil(object): |
cmd.extend(args) |
return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout) |
- def check_call(self, *args): |
+ def check_call(self, *args, **kwargs): |
+ def tee(stream, buf): |
+ def _inner(line): |
Ryan Tseng
2014/04/18 00:28:53
"data" or "char" is probably more appropriate vs "
iannucci
2014/04/28 21:05:28
What? That's not true...
/me reads subprocess2 im
|
+ stream.write(line) |
+ buf.append(line) |
+ return _inner |
+ |
cmd = [sys.executable, self.path] |
if self.bypass_prodaccess: |
cmd.append('--bypass_prodaccess') |
cmd.extend(args) |
- ((out, err), code) = subprocess2.communicate( |
- cmd, |
- stdout=subprocess2.PIPE, |
- stderr=subprocess2.PIPE, |
- env=self.get_sub_env(), |
- timeout=self.timeout) |
+ |
+ out = [] |
+ err = [] |
Ryan Tseng
2014/04/18 00:28:53
why not a string or cStringIO?
I'd probably prefe
Ryan Tseng
2014/04/18 00:37:36
Disregard that, I ran a test and your list impleme
iannucci
2014/04/28 21:05:28
string allocation n' stuff
|
+ |
+ proc = subprocess2.Popen(cmd, env=self.get_sub_env(), |
+ stdout=subprocess2.PIPE, |
+ stderr=subprocess2.PIPE) |
+ if kwargs.get('verbose'): |
+ proc.stdout_cb = tee(sys.stdout, out) |
+ proc.stderr_cb = tee(sys.stderr, err) |
+ else: |
+ proc.stdout_cb = out.append |
+ proc.stderr_cb = err.append |
+ proc.communicate(timeout=self.timeout) |
+ code = proc.returncode |
+ |
+ out = ''.join(out) |
+ err = ''.join(err) |
# Parse output. |
status_code_match = re.search('status=([0-9]+)', err) |