[NaCl SDK] Change build script to test sdk_updater, also push manifest snippets per bot.

native_client_sdk/src/build_tools/sdk_tools/sdk_update.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 '''A simple tool to update the Native Client SDK to the latest version'''
 
 import cStringIO
 import cygtar
 import errno
@@ skipping 85 matching lines @@
     mgs: A string to send to stderr.'''
   sys.stderr.write("WARNING: %s\n" % msg)
   sys.stderr.flush()
 
 
 class Error(Exception):
   '''Generic error/exception for sdk_update module'''
   pass
 
 
-def GetHostOS():
-  '''Returns the host_os value that corresponds to the current host OS'''
-  return {
-      'linux2': 'linux',
-      'darwin': 'mac',
-      'cygwin': 'win',
-      'win32':  'win'
-      }[sys.platform]
-
 def UrlOpen(url):
   request = fancy_urllib.FancyRequest(url)
   ca_certs = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                           'cacerts.txt')
   request.set_ssl_info(ca_certs=ca_certs)
   url_opener = urllib2.build_opener(
       fancy_urllib.FancyProxyHandler(),
       fancy_urllib.FancyRedirectHandler(),
       fancy_urllib.FancyHTTPSHandler())
   return url_opener.open(request)
@@ skipping 75 matching lines @@
       # handling a Windows flaky access error.  Sleep one second and try
       # again.
       time.sleep(num_tries + 1)
   # end of while loop -- could not RenameDir
   raise Error('Could not RenameDir %s => %s after %d tries.\n' %
               'Please check that no shells or applications '
               'are accessing files in %s.'
               % (srcdir, destdir, num_tries, destdir))
 
 
-def ShowProgress(progress):
-  ''' A download-progress function used by class Archive.
-      (See DownloadAndComputeHash).'''
-  global count  # A divider, so we don't emit dots too often.
-
-  if progress == 0:
-    count = 0
-  elif progress == 100:
-    sys.stdout.write('\n')
-  else:
-    count = count + 1
-    if count > 10:
-      sys.stdout.write('.')
-      sys.stdout.flush()
-      count = 0
-
-
 class ProgressFunction(object):
   '''Create a progress function for a file with a given size'''
 
   def __init__(self, file_size=0):
     '''Constructor
 
     Args:
       file_size: number of bytes in file.  0 indicates unknown'''
     self.dots = 0
     self.file_size = int(file_size)
 
   def GetProgressFunction(self):
     '''Returns a progress function based on a known file size'''
     def ShowKnownProgress(progress):
       if progress == 0:
         sys.stdout.write('|%s|\n' % ('=' * 48))
       else:
         new_dots = progress * 50 / self.file_size - self.dots
         sys.stdout.write('.' * new_dots)
         self.dots += new_dots
         if progress == self.file_size:
           sys.stdout.write('\n')
       sys.stdout.flush()
 
     return ShowKnownProgress
 
 
-def DownloadAndComputeHash(from_stream, to_stream=None, progress_func=None):
-  ''' Download the archive data from from-stream and generate sha1 and
-      size info.
+def DownloadArchiveToFile(archive, dest_path):
+  '''Download the archive's data to a file at dest_path.
+
+  As a side effect, computes the sha1 hash and data size, both returned as a
+  tuple. Raises an Error if the url can't be opened, or an IOError exception if
+  dest_path can't be opened.
 
   Args:
-    from_stream:   An input stream that supports read.
-    to_stream:     [optional] the data is written to to_stream if it is
-                   provided.
-    progress_func: [optional] A function used to report download progress. If
-                   provided, progress_func is called with progress=0 at the
-                   beginning of the download, periodically with progress=1
-                   during the download, and progress=100 at the end.
-
-  Return
-    A tuple (sha1, size) where sha1 is a sha1-hash for the archive data and
-    size is the size of the archive data in bytes.'''
-  # Use a no-op progress function if none is specified.
-  def progress_no_op(progress):
-    pass
-  if not progress_func:
-    progress_func = progress_no_op
-
-  sha1_hash = hashlib.sha1()
+    dest_path: Path for the file that will receive the data.
+  Return:
+    A tuple (sha1, size) with the sha1 hash and data size respectively.'''
+  sha1 = None
   size = 0
-  progress_func(progress=0)
-  while(1):
-    data = from_stream.read(32768)
-    if not data:
-      break
-    sha1_hash.update(data)
-    size += len(data)
-    if to_stream:
-      to_stream.write(data)
-    progress_func(size)
-
-  progress_func(progress=100)
-  return sha1_hash.hexdigest(), size
+  with open(dest_path, 'wb') as to_stream:
+    from_stream = None
+    try:
+      from_stream = UrlOpen(archive.url)
+    except urllib2.URLError:
+      raise Error('Cannot open "%s" for archive %s' %
+          (archive.url, archive.host_os))
+    try:
+      content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH])
+      progress_function = ProgressFunction(content_length).GetProgressFunction()
+      InfoPrint('Downloading %s' % archive.url)
+      sha1, size = manifest_util.DownloadAndComputeHash(
+          from_stream,
+          to_stream=to_stream,
+          progress_func=progress_function)
+      if size != content_length:
+        raise Error('Download size mismatch for %s.\n'
+                    'Expected %s bytes but got %s' %
+                    (archive.url, content_length, size))
+    finally:
+      if from_stream: from_stream.close()
+  return sha1, size
 
 
 def LoadManifestFromFile(path):
   '''Returns a manifest loaded from the JSON file at |path|.
 
   If the path does not exist or is invalid, returns an empty manifest.'''
   if not os.path.exists(path):
     return manifest_util.SDKManifest()
 
   with open(path, 'r') as f:
     json_string = f.read()
   if not json_string:
     return manifest_util.SDKManifest()
 
   manifest = manifest_util.SDKManifest()
   manifest.LoadManifestString(json_string)
   return manifest
 
 
 def LoadManifestFromURL(url):
   '''Returns a manifest loaded from |url|.'''
   try:
     url_stream = UrlOpen(url)
   except urllib2.URLError as e:
     raise Error('Unable to open %s. [%s]' % (url, e))
 
   manifest_stream = cStringIO.StringIO()
-  sha1, size = DownloadAndComputeHash(
-      url_stream, manifest_stream)
+  sha1, size = manifest_util.DownloadAndComputeHash(url_stream, manifest_stream)
   manifest = manifest_util.SDKManifest()
   manifest.LoadManifestString(manifest_stream.getvalue())
 
   def BundleFilter(bundle):
     # Only add this bundle if it's supported on this platform.
-    return bundle.GetArchive(GetHostOS())
+    return bundle.GetHostOSArchive()
 
   manifest.FilterBundles(BundleFilter)
   return manifest
 
 
 def WriteManifestToFile(manifest, path):
   '''Write |manifest| to a JSON file at |path|.'''
   json_string = manifest.GetManifestString()
 
   # Write the JSON data to a temp file.
   temp_file_name = None
   # TODO(dspringer): Use file locks here so that multiple sdk_updates can
   # run at the same time.
   with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
     f.write(json_string)
     temp_file_name = f.name
   # Move the temp file to the actual file.
   if os.path.exists(path):
     os.remove(path)
   shutil.move(temp_file_name, path)
 
 
-def DownloadArchiveToFile(archive, dest_path):
-  '''Download the archive's data to a file at dest_path.
-
-  As a side effect, computes the sha1 hash and data size, both returned as a
-  tuple. Raises an Error if the url can't be opened, or an IOError exception if
-  dest_path can't be opened.
-
-  Args:
-    dest_path: Path for the file that will receive the data.
-  Return:
-    A tuple (sha1, size) with the sha1 hash and data size respectively.'''
-  sha1 = None
-  size = 0
-  with open(dest_path, 'wb') as to_stream:
-    from_stream = None
-    try:
-      from_stream = UrlOpen(archive.url)
-    except urllib2.URLError:
-      raise Error('Cannot open "%s" for archive %s' %
-          (archive.url, archive.host_os))
-    try:
-      content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH])
-      progress_function = ProgressFunction(content_length).GetProgressFunction()
-      InfoPrint('Downloading %s' % archive.url)
-      sha1, size = DownloadAndComputeHash(
-          from_stream,
-          to_stream=to_stream,
-          progress_func=progress_function)
-      if size != content_length:
-        raise Error('Download size mismatch for %s.\n'
-                    'Expected %s bytes but got %s' %
-                    (archive.url, content_length, size))
-    finally:
-      if from_stream: from_stream.close()
-  return sha1, size
-
-
 #------------------------------------------------------------------------------
 # Commands
 
 
 def List(options, argv):
   '''Usage: %prog [options] list
 
   Lists the available SDK bundles that are available for download.'''
   def PrintBundles(bundles):
     for bundle in bundles:
@@ skipping 69 matching lines @@
 
   for bundle in bundles:
     bundle_path = os.path.join(options.sdk_root_dir, bundle.name)
     bundle_update_path = '%s_update' % bundle_path
     if not (bundle.name in args or
             ALL in args or (RECOMMENDED in args and
                             bundle[RECOMMENDED] == 'yes')):
       continue
     def UpdateBundle():
       '''Helper to install a bundle'''
-      archive = bundle.GetArchive(GetHostOS())
+      archive = bundle.GetHostOSArchive()
       (scheme, host, path, _, _, _) = urlparse.urlparse(archive['url'])
       dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1])
       sha1, size = DownloadArchiveToFile(archive, dest_filename)
       if sha1 != archive.GetChecksum():
         raise Error("SHA1 checksum mismatch on '%s'.  Expected %s but got %s" %
                     (bundle.name, archive.GetChecksum(), sha1))
       if size != archive.size:
         raise Error("Size mismatch on Archive.  Expected %s but got %s bytes" %
                     (archive.size, size))
       InfoPrint('Updating bundle %s to version %s, revision %s' % (
@@ skipping 126 matching lines @@
 
   return 0  # Success
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except Error as error:
     print "Error: %s" % error
     sys.exit(1)