Modify get_archive script to optionally download debug Dartium

tools/get_archive.py

 #!/usr/bin/env python
 #
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 # Gets or updates a content shell (a nearly headless build of chrome). This is
 # used for running browser tests of client applications.
 
 import json
@@ skipping 14 matching lines @@
 # Change into the dart directory as we want the project to be rooted here.
 dart_src = NormJoin(os.path.dirname(sys.argv[0]), os.pardir)
 os.chdir(dart_src)
 
 GSUTIL_DIR = os.path.join('third_party', 'gsutil')
 GSUTIL = GSUTIL_DIR + '/gsutil'
 
 DRT_DIR = os.path.join('client', 'tests', 'drt')
 DRT_VERSION = os.path.join(DRT_DIR, 'LAST_VERSION')
 DRT_LATEST_PATTERN = (
-    'gs://dartium-archive/latest/drt-%(osname)s-inc-*.zip')
-DRT_PERMANENT_PATTERN = ('gs://dartium-archive/drt-%(osname)s-inc/drt-'
-                         '%(osname)s-inc-%(num1)s.%(num2)s.zip')
+    'gs://dartium-archive/latest/drt-%(osname)s-%(bot)s-*.zip')
+DRT_PERMANENT_PATTERN = ('gs://dartium-archive/drt-%(osname)s-%(bot)s/drt-'
+                         '%(osname)s-%(bot)s-%(num1)s.%(num2)s.zip')
 
 DARTIUM_DIR = os.path.join('client', 'tests', 'dartium')
 DARTIUM_VERSION = os.path.join(DARTIUM_DIR, 'LAST_VERSION')
 DARTIUM_LATEST_PATTERN = (
-    'gs://dartium-archive/latest/dartium-%(osname)s-inc-*.zip')
-DARTIUM_PERMANENT_PATTERN = ('gs://dartium-archive/dartium-%(osname)s-inc/'
-                             'dartium-%(osname)s-inc-%(num1)s.%(num2)s.zip')
+    'gs://dartium-archive/latest/dartium-%(osname)s-%(bot)s-*.zip')
+DARTIUM_PERMANENT_PATTERN = ('gs://dartium-archive/dartium-%(osname)s-%(bot)s/'
+                             'dartium-%(osname)s-%(bot)s-%(num1)s.%(num2)s.zip')
 
 CHROMEDRIVER_DIR = os.path.join('tools', 'testing', 'dartium-chromedriver')
 CHROMEDRIVER_VERSION = os.path.join(CHROMEDRIVER_DIR, 'LAST_VERSION')
 CHROMEDRIVER_LATEST_PATTERN = (
-    'gs://dartium-archive/latest/chromedriver-%(osname)s-inc-*.zip')
+    'gs://dartium-archive/latest/chromedriver-%(osname)s-%(bot)s-*.zip')
 CHROMEDRIVER_PERMANENT_PATTERN = ('gs://dartium-archive/chromedriver-%(osname)s'
-                                  '-inc/chromedriver-%(osname)s-inc-%(num1)s.'
+                                  '-%(bot)s/chromedriver-%(osname)s-%(bot)s-%(num1)s.'
                                   '%(num2)s.zip')
 
 SDK_DIR = os.path.join(utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
     'dart-sdk')
 SDK_VERSION = os.path.join(SDK_DIR, 'LAST_VERSION')
 SDK_LATEST_PATTERN = 'gs://dart-editor-archive-continuous/latest/VERSION'
 # TODO(efortuna): Once the x64 VM also is optimized, select the version
 # based on whether we are running on a 32-bit or 64-bit system.
-SDK_PERMANENT = ('gs://dart-editor-archive-continuous/%(version_num)s/' + 
+SDK_PERMANENT = ('gs://dart-editor-archive-continuous/%(version_num)s/' +
     'dartsdk-%(osname)s-32.zip')
 
 # Dictionary storing the earliest revision of each download we have stored.
 LAST_VALID = {'dartium': 4285, 'chromedriver': 7823, 'sdk': 9761, 'drt': 5342}
 
 sys.path.append(os.path.join(GSUTIL_DIR, 'boto'))
 import boto
 
 
 def ExecuteCommand(*cmd):
@@ skipping 46 matching lines @@
 * You need to do a one-time configuration step to access Google Storage.
 * Please run this command and follow the instructions:
 *     %s config
 *
 * NOTE: When prompted you can leave "project-id" blank. Just hit enter.
 *******************************************************************************
 ''' % GSUTIL
     sys.exit(1)
 
 
-def GetDartiumRevision(name, directory, version_file, latest_pattern,
+def GetDartiumRevision(name, bot, directory, version_file, latest_pattern,
     permanent_prefix, revision_num=None):
   """Get the latest binary that is stored in the dartium archive.
 
   Args:
     name: the name of the desired download.
     directory: target directory (recreated) to install binary
     version_file: name of file with the current version stamp
     latest_pattern: the google store url pattern pointing to the latest binary
     permanent_prefix: stable google store folder used to download versions
     revision_num: The desired revision number to retrieve. If revision_num is
         None, we return the latest revision. If the revision number is specified
         but unavailable, find the nearest older revision and use that instead.
   """
   osdict = {'Darwin':'mac', 'Linux':'lucid64', 'Windows':'win'}
 
   def FindPermanentUrl(out, osname, revision_num):
     output_lines = out.split()
     latest = output_lines[-1]
     if not revision_num:
       revision_num = latest[latest.rindex('-') + 1 : latest.index('.')]
       latest = (permanent_prefix[:permanent_prefix.rindex('/')] % { 'osname' :
-          osname } + latest[latest.rindex('/'):])
+          osname, 'bot' : bot } + latest[latest.rindex('/'):])
     else:
       latest = (permanent_prefix % { 'osname' : osname, 'num1' : revision_num,
-          'num2' : revision_num })
+          'num2' : revision_num, 'bot' : bot })
       foundURL = False
       while not foundURL:
         # Test to ensure this URL exists because the dartium-archive builds can
         # have unusual numbering (a range of CL numbers) sometimes.
         result, out = Gsutil('ls', permanent_prefix % {'osname' : osname,
-            'num1': revision_num, 'num2': '*' })
+            'num1': revision_num, 'num2': '*', 'bot': bot })
         if result == 0:
           # First try to find one with the the second number the same as the
           # requested number.
           latest = out.split()[0]
           # Now test that the permissions are correct so you can actually
           # download it.
           temp_dir = tempfile.mkdtemp()
           temp_zip = os.path.join(temp_dir, 'foo.zip')
           returncode, out = Gsutil('cp', latest, 'file://' + temp_zip)
           if returncode == 0:
             foundURL = True
           else:
             # Unable to download this item (most likely because something went
             # wrong on the upload and the permissions are bad). Keep looking for
             # a different URL.
             revision_num = int(revision_num) - 1
           shutil.rmtree(temp_dir)
         else:
           # Now try to find one with a nearby CL num.
           revision_num = int(revision_num) - 1
           if revision_num <= 0:
             TooEarlyError()
     return latest
-    
-  GetFromGsutil(name, directory, version_file, latest_pattern, osdict, 
-                  FindPermanentUrl, revision_num)
+
+  GetFromGsutil(name, directory, version_file, latest_pattern, osdict,
+                  FindPermanentUrl, revision_num, bot)
 
 
 def GetSdkRevision(name, directory, version_file, latest_pattern,
     permanent_prefix, revision_num):
   """Get a revision of the SDK from the editor build archive.
 
   Args:
     name: the name of the desired download
     directory: target directory (recreated) to install binary
     version_file: name of file with the current version stamp
@@ skipping 14 matching lines @@
       version_info = temp_file.read()
       temp_file.close()
       os.unlink(temp_file.name)
       if version_info != '':
         rev_num = json.loads(version_info)['revision']
       else:
         print 'Unable to get latest version information.'
         return ''
     latest = (permanent_prefix % { 'osname' : osname, 'version_num': rev_num})
     return latest
-    
+
   GetFromGsutil(name, directory, version_file, latest_pattern, osdict,
                   FindPermanentUrl, revision_num)
 
 
 def GetFromGsutil(name, directory, version_file, latest_pattern,
-    os_name_dict, get_permanent_url, revision_num = ''):
+    os_name_dict, get_permanent_url, revision_num = '', bot = None):
   """Download and unzip the desired file from Google Storage.
     Args:
     name: the name of the desired download
     directory: target directory (recreated) to install binary
     version_file: name of file with the current version stamp
     latest_pattern: the google store url pattern pointing to the latest binary
     os_name_dict: a dictionary of operating system names and their corresponding
         strings on the google storage site.
     get_permanent_url: a function that accepts a listing of available files
         and the os name, and returns a permanent URL for downloading.
     revision_num: the desired revision number to get (if not supplied, we get
         the latest revision)
   """
   system = platform.system()
   try:
     osname = os_name_dict[system]
   except KeyError:
     print >>sys.stderr, ('WARNING: platform "%s" does not support'
         '%s.') % (system, name)
     return 0
 
   EnsureConfig()
 
-  # Query for the lastest version
-  pattern = latest_pattern  % { 'osname' : osname }
+  # Query for the latest version
+  pattern = latest_pattern  % { 'osname' : osname, 'bot' : bot }
   result, out = Gsutil('ls', pattern)
   if result == 0:
     # use permanent link instead, just in case the latest zip entry gets deleted
     # while we are downloading it.
     latest = get_permanent_url(out, osname, revision_num)
   else: # e.g. no access
     print "Couldn't download %s: %s\n%s" % (name, pattern, out)
     if not os.path.exists(version_file):
       print "Using %s will not work. Please try again later." % name
     return 0
@@ skipping 62 matching lines @@
     return
   shutil.copy('third_party/drt_resources/AHEM____.TTF', drt_dir)
 
 
 def main():
   parser = optparse.OptionParser(usage='usage: %prog [options] download_name')
   parser.add_option('-r', '--revision', dest='revision',
                     help='Desired revision number to retrieve for the SDK. If '
                     'unspecified, retrieve the latest SDK build.',
                     action='store', default=None)
+  parser.add_option('-d', '--debug', dest='debug',
+                    help='Download a debug archive instead of a release.',
+                    action='store_true', default=False)
   args, positional = parser.parse_args()
 
   if args.revision and int(args.revision) < LAST_VALID[positional[0]]:
     return TooEarlyError()
 
+  # Use the incremental release bot ('dartium-*-inc') by default.
+  bot = 'inc'
+  if args.debug:
+    bot = 'debug'
+
   if positional[0] == 'dartium':
-    GetDartiumRevision('Dartium', DARTIUM_DIR, DARTIUM_VERSION,
+    GetDartiumRevision('Dartium', bot, DARTIUM_DIR, DARTIUM_VERSION,
                          DARTIUM_LATEST_PATTERN, DARTIUM_PERMANENT_PATTERN,
                          args.revision)
   elif positional[0] == 'chromedriver':
-    GetDartiumRevision('chromedriver', CHROMEDRIVER_DIR, CHROMEDRIVER_VERSION,
+    GetDartiumRevision('chromedriver', bot, CHROMEDRIVER_DIR, CHROMEDRIVER_VERSION,
                          CHROMEDRIVER_LATEST_PATTERN,
                          CHROMEDRIVER_PERMANENT_PATTERN, args.revision)
   elif positional[0] == 'sdk':
     GetSdkRevision('sdk', SDK_DIR, SDK_VERSION, SDK_LATEST_PATTERN,
         SDK_PERMANENT, args.revision)
   elif positional[0] == 'drt':
-    GetDartiumRevision('content_shell', DRT_DIR, DRT_VERSION,
+    GetDartiumRevision('content_shell', bot, DRT_DIR, DRT_VERSION,
                          DRT_LATEST_PATTERN, DRT_PERMANENT_PATTERN,
                          args.revision)
     CopyDrtFont(DRT_DIR)
   else:
     print ('Please specify the target you wish to download from Google Storage '
         '("drt", "dartium", "chromedriver", or "sdk")')
 
 if __name__ == '__main__':
   sys.exit(main())