Implement git-drover.

download_from_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Download files from Google Storage based on SHA1 sums."""
 
 
 import hashlib
 import optparse
@@ skipping 48 matching lines @@
     if not os.path.exists(path):
       raise FileNotFoundError('GSUtil not found in %s' % path)
     self.path = path
     self.timeout = timeout
     self.boto_path = boto_path
     self.bypass_prodaccess = bypass_prodaccess
 
   def get_sub_env(self):
     env = os.environ.copy()
     if self.boto_path == os.devnull:
-      env['AWS_CREDENTIAL_FILE'] = ''
+      env.pop('AWS_CREDENTIAL_FILE', None)

[Ryan Tseng, 2014/04/18 00:28:53]

???

[iannucci, 2014/04/28 21:05:28]

Otherwise gsutil freaks out because the env var is set but not a valid path.
This is more-better.

       env['BOTO_CONFIG'] = ''
     elif self.boto_path:
       env['AWS_CREDENTIAL_FILE'] = self.boto_path
       env['BOTO_CONFIG'] = self.boto_path
     else:
       custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools'
       custompath = os.path.expanduser(custompath)
       if os.path.exists(custompath):
         env['AWS_CREDENTIAL_FILE'] = custompath
 
     return env
 
   def call(self, *args):
     cmd = [sys.executable, self.path]
     if self.bypass_prodaccess:
       cmd.append('--bypass_prodaccess')
     cmd.extend(args)
     return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)
 
-  def check_call(self, *args):
+  def check_call(self, *args, **kwargs):
+    def tee(stream, buf):
+      def _inner(line):

[Ryan Tseng, 2014/04/18 00:28:53]

"data" or "char" is probably more appropriate vs "line", since subprocess2 will
only ever call this with a single char

[iannucci, 2014/04/28 21:05:28]

What? That's not true...

/me reads subprocess2 implementation

What the... who... why.... arg! :(

+        stream.write(line)
+        buf.append(line)
+      return _inner
+
     cmd = [sys.executable, self.path]
     if self.bypass_prodaccess:
       cmd.append('--bypass_prodaccess')
     cmd.extend(args)
-    ((out, err), code) = subprocess2.communicate(
-        cmd,
-        stdout=subprocess2.PIPE,
-        stderr=subprocess2.PIPE,
-        env=self.get_sub_env(),
-        timeout=self.timeout)
+
+    out = []
+    err = []

[Ryan Tseng, 2014/04/18 00:28:53]

why not a string or cStringIO?

I'd probably prefer cStringIO.

[Ryan Tseng, 2014/04/18 00:37:36]

Disregard that, I ran a test and your list implementation is faster.

I don't understand python.

[iannucci, 2014/04/28 21:05:28]

string allocation n' stuff

+
+    proc = subprocess2.Popen(cmd, env=self.get_sub_env(),
+                             stdout=subprocess2.PIPE,
+                             stderr=subprocess2.PIPE)
+    if kwargs.get('verbose'):
+      proc.stdout_cb = tee(sys.stdout, out)
+      proc.stderr_cb = tee(sys.stderr, err)
+    else:
+      proc.stdout_cb = out.append
+      proc.stderr_cb = err.append
+    proc.communicate(timeout=self.timeout)
+    code = proc.returncode
+
+    out = ''.join(out)
+    err = ''.join(err)
 
     # Parse output.
     status_code_match = re.search('status=([0-9]+)', err)
     if status_code_match:
       return (int(status_code_match.group(1)), out, err)
     if ('You are attempting to access protected data with '
           'no configured credentials.' in err):
       return (403, out, err)
     if 'No such object' in err:
       return (404, out, err)
@@ skipping 348 matching lines @@
     return code
 
   return download_from_google_storage(
       input_filename, base_url, gsutil, options.num_threads, options.directory,
       options.recursive, options.force, options.output, options.ignore_errors,
       options.sha1_file, options.verbose, options.auto_platform)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))