Made get_archive.py resilient to 403 forbidden errors.

tools/get_archive.py

 #!/usr/bin/env python
 #
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 # Gets or updates a DumpRenderTree (a nearly headless build of chrome). This is
 # used for running browser tests of client applications.
 
 import json
@@ skipping 53 matching lines @@
 LAST_VALID = {'dartium': 4285, 'chromedriver': 7823, 'sdk': 9761, 'drt': 5342}
 
 sys.path.append(os.path.join(GSUTIL_DIR, 'boto'))
 import boto
 
 
 def ExecuteCommand(*cmd):
   """Execute a command in a subprocess."""
   pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
   output, error = pipe.communicate()
-  return pipe.returncode, output
+  return pipe.returncode, output, error
 
 
 def ExecuteCommandVisible(*cmd):
   """Execute a command in a subprocess, but show stdout/stderr."""
   result = subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr,
                            stdin=sys.stdin)
   if result != 0:
     raise Exception('Execution of "%s" failed' % ' '.join(cmd))
 
 
@@ skipping 62 matching lines @@
       revision_num = latest[latest.rindex('-') + 1 : latest.index('.')]
       latest = (permanent_prefix[:permanent_prefix.rindex('/')] % { 'osname' :
           osname } + latest[latest.rindex('/'):])
     else:
       latest = (permanent_prefix % { 'osname' : osname, 'num1' : revision_num,
           'num2' : revision_num })
       foundURL = False
       while not foundURL:
         # Test to ensure this URL exists because the dartium-archive builds can
         # have unusual numbering (a range of CL numbers) sometimes.
-        result, out = Gsutil('ls', permanent_prefix % {'osname' : osname,
+        result, out, _ = Gsutil('ls', permanent_prefix % {'osname' : osname,
             'num1': '*', 'num2': revision_num })
         if result == 0:
           # First try to find one with the the second number the same as the
           # requested number.
           latest = out.split()[0]
-          foundURL = True
+          # Now test that the permissions are correct so you can actually
+          # download it.
+          temp_dir = tempfile.mkdtemp()
+          temp_zip = os.path.join(temp_dir, 'foo.zip')
+          returncode, out, err = Gsutil('cp', latest, 'file://' + temp_zip)

[vsm, 2012/08/22 18:08:09]

It doesn't look like you're using err here.

+          if returncode == 0:
+            foundURL = True
+          else:
+            # Unable to download this item (most likely because something went
+            # wrong on the upload and the permissions are bad). Keep looking for
+            # a different URL.
+            revision_num = int(revision_num) - 1
+          shutil.rmtree(temp_dir)
         else:
           # Now try to find one with a nearby CL num.
           revision_num = int(revision_num) - 1
           if revision_num <= 0:
             TooEarlyError()
     return latest
     
   GetFromGsutil(name, directory, version_file, latest_pattern, osdict, 
                   FindPermanentUrl, revision_num)
 
@@ skipping 52 matching lines @@
     osname = os_name_dict[system]
   except KeyError:
     print >>sys.stderr, ('WARNING: platform "%s" does not support'
         '%s.') % (system, name)
     return 0
 
   EnsureConfig()
 
   # Query for the lastest version
   pattern = latest_pattern  % { 'osname' : osname }
-  result, out = Gsutil('ls', pattern)
+  result, out, _ = Gsutil('ls', pattern)
   if result == 0:
     # use permanent link instead, just in case the latest zip entry gets deleted
     # while we are downloading it.
     latest = get_permanent_url(out, osname, revision_num)
   else: # e.g. no access
     print "Couldn't download %s: %s\n%s" % (name, pattern, out)
     if not os.path.exists(version_file):
       print "Using %s will not work. Please try again later." % name
     return 0
 
@@ skipping 14 matching lines @@
   try:
     temp_zip = os.path.join(temp_dir, 'drt.zip')
     temp_zip_url = 'file://' + temp_zip
     # It's nice to show download progress
     GsutilVisible('cp', latest, temp_zip_url)
 
     if platform.system() != 'Windows':
       # The Python zip utility does not preserve executable permissions, but
       # this does not seem to be a problem for Windows, which does not have a
       # built in zip utility. :-/
-      result, out = ExecuteCommand('unzip', temp_zip, '-d', temp_dir)
+      result, out, _ = ExecuteCommand('unzip', temp_zip, '-d', temp_dir)
       if result != 0:
         raise Exception('Execution of "unzip %s -d %s" failed: %s' %
                         (temp_zip, temp_dir, str(out)))
       unzipped_dir = temp_dir + '/' + os.path.basename(latest)[:-len('.zip')]
     else:
       z = zipfile.ZipFile(temp_zip)
       z.extractall(temp_dir)
       unzipped_dir = os.path.join(temp_dir,
                                   os.path.basename(latest)[:-len('.zip')])
       z.close()
@@ skipping 46 matching lines @@
   elif positional[0] == 'drt':
     GetDartiumRevision('DumpRenderTree', DRT_DIR, DRT_VERSION,
                          DRT_LATEST_PATTERN, DRT_PERMANENT_PATTERN,
                          args.revision)
   else:
     print ('Please specify the target you wish to download from Google Storage '
         '("drt", "dartium", "chromedriver", or "sdk")')
 
 if __name__ == '__main__':
   sys.exit(main())