[NaCl SDK] Change build script to test sdk_updater, also push manifest snippets per bot.

native_client_sdk/src/build_tools/build_sdk.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Entry point for both build and try bots
 
 This script is invoked from XXX, usually without arguments
 to package an SDK. It automatically determines whether
 this SDK is for mac, win, linux.
 
 The script inspects the following environment variables:
 
 BUILDBOT_BUILDERNAME to determine whether the script is run locally
 and whether it should upload an SDK to file storage (GSTORE)
 """
 
 
 # std python includes
+import multiprocessing
 import optparse
 import os
 import platform
+import subprocess
 import sys
 
 # local includes
 import buildbot_common
 import build_utils
 import lastchange
+import manifest_util
 
 # Create the various paths of interest
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
 SDK_EXAMPLE_DIR = os.path.join(SDK_SRC_DIR, 'examples')
 SDK_DIR = os.path.dirname(SDK_SRC_DIR)
 SRC_DIR = os.path.dirname(SDK_DIR)
 NACL_DIR = os.path.join(SRC_DIR, 'native_client')
 OUT_DIR = os.path.join(SRC_DIR, 'out')
 PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
+SERVER_DIR = os.path.join(OUT_DIR, 'local_server')
 
 
 # Add SDK make tools scripts to the python path.
 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
 sys.path.append(os.path.join(NACL_DIR, 'build'))
 
 import getos
 import http_download
 import oshelpers
 
 GSTORE = 'http://commondatastorage.googleapis.com/nativeclient-mirror/nacl/'
 MAKE = 'nacl_sdk/make_3_81/make.exe'
 CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')
 
 
+def HTTPServerProcess(conn, serve_dir):
+  """Run a local httpserver with a randomly-chosen port.
+
+  This function assumes it is run as a child process using multiprocessing.
+
+  Args:
+    conn: A connection to the parent process. The child process sends
+        the local port, and waits for a message from the parent to
+        stop serving.
+    serve_dir: The directory to serve. All files are accessible through
+       http://localhost:<port>/path/to/filename.
+  """
+  import BaseHTTPServer
+  import SimpleHTTPServer
+
+  os.chdir(serve_dir)
+  httpd = BaseHTTPServer.HTTPServer(('', 0),
+                                    SimpleHTTPServer.SimpleHTTPRequestHandler)
+  conn.send(httpd.server_address[1])  # the chosen port number
+  httpd.timeout = 0.5  # seconds
+  running = True
+  while running:
+    httpd.handle_request()
+    if conn.poll():
+      running = conn.recv()
+  conn.close()
+
+
+class LocalHTTPServer(object):
+  """Class to start a local HTTP server as a child process."""
+
+  def __init__(self, serve_dir):
+    parent_conn, child_conn = multiprocessing.Pipe()
+    self.process = multiprocessing.Process(target=HTTPServerProcess,
+                                           args=(child_conn, serve_dir))
+    self.process.start()
+    if parent_conn.poll(1):  # wait a second
+      self.port = parent_conn.recv()
+    else:
+      raise Exception('Unable to launch HTTP server.')
+
+    self.conn = parent_conn
+
+  def Shutdown(self):
+    """Send a message to the child HTTP server process and wait for it to
+        finish."""
+    self.conn.send(False)
+    self.process.join()
+
+  def GetURL(self, rel_url):
+    """Get the full url for a file on the local HTTP server.
+
+    Args:
+      rel_url: A URL fragment to convert to a full URL. For example,
+          GetURL('foobar.baz') -> 'http://localhost:1234/foobar.baz'
+    """
+    return 'http://localhost:%d/%s' % (self.port, rel_url)
+
+
 def AddMakeBat(pepperdir, makepath):
   """Create a simple batch file to execute Make.
-  
+
   Creates a simple batch file named make.bat for the Windows platform at the
   given path, pointing to the Make executable in the SDK."""
-  
+
   makepath = os.path.abspath(makepath)
   if not makepath.startswith(pepperdir):
     buildbot_common.ErrorExit('Make.bat not relative to Pepper directory: ' +
                               makepath)
-  
+
   makeexe = os.path.abspath(os.path.join(pepperdir, 'tools'))
   relpath = os.path.relpath(makeexe, makepath)
 
   fp = open(os.path.join(makepath, 'make.bat'), 'wb')
   outpath = os.path.join(relpath, 'make.exe')
 
   # Since make.bat is only used by Windows, for Windows path style
   outpath = outpath.replace(os.path.sep, '\\')
   fp.write('@%s %%*\n' % outpath)
   fp.close()
@@ skipping 120 matching lines @@
   ppapi = os.path.join(tc_dst_inc, 'ppapi')
   buildbot_common.RemoveDir(ppapi)
 
   # Copy in c and c/dev headers
   buildbot_common.MakeDir(os.path.join(ppapi, 'c', 'dev'))
   buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'c', '*.h'),
           os.path.join(ppapi, 'c'))
   buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'c', 'dev', '*.h'),
           os.path.join(ppapi, 'c', 'dev'))
 
-  # Run the generator to overwrite IDL files  
+  # Run the generator to overwrite IDL files
   buildbot_common.Run([sys.executable, 'generator.py', '--wnone', '--cgen',
        '--release=M' + pepper_ver, '--verbose', '--dstroot=%s/c' % ppapi],
        cwd=os.path.join(PPAPI_DIR, 'generators'))
 
   # Remove private and trusted interfaces
   buildbot_common.RemoveDir(os.path.join(ppapi, 'c', 'private'))
   buildbot_common.RemoveDir(os.path.join(ppapi, 'c', 'trusted'))
 
   # Copy in the C++ headers
   buildbot_common.MakeDir(os.path.join(ppapi, 'cpp', 'dev'))
@@ skipping 36 matching lines @@
   tcname = platform + '_' + arch
   tmpdir = os.path.join(SRC_DIR, 'out', 'tc_temp')
   buildbot_common.RemoveDir(tmpdir)
   buildbot_common.MakeDir(tmpdir)
 
   if 'newlib' in toolchains:
     # Untar the newlib toolchains
     tarfile = GetNewlibToolchain(platform, arch)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                         cwd=NACL_DIR)
-  
+
     # Then rename/move it to the pepper toolchain directory
     srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk')
     newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
     buildbot_common.Move(srcdir, newlibdir)
 
   if 'glibc' in toolchains:
     # Untar the glibc toolchains
     tarfile = GetGlibcToolchain(platform, arch)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                         cwd=NACL_DIR)
-  
+
     # Then rename/move it to the pepper toolchain directory
     srcdir = os.path.join(tmpdir, 'toolchain', tcname)
     glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
     buildbot_common.Move(srcdir, glibcdir)
 
   # Untar the pnacl toolchains
   if 'pnacl' in toolchains:
     tmpdir = os.path.join(tmpdir, 'pnacl')
     buildbot_common.RemoveDir(tmpdir)
     buildbot_common.MakeDir(tmpdir)
@@ skipping 58 matching lines @@
     buildbot_common.ErrorExit('Missing arch %s' % arch)
 
 
 EXAMPLE_MAP = {
   'newlib': [
     'debugging',
     'file_histogram',
     'fullscreen_tumbler',
     'gamepad',
     'geturl',
-    'hello_world_interactive', 
+    'hello_world_interactive',
     'hello_world_newlib',
     'input_events',
     'load_progress',
     'mouselock',
     'multithreaded_input_events',
     'pi_generator',
     'pong',
     'sine_synth',
     'tumbler',
     'websocket'
@@ skipping 88 matching lines @@
   buildbot_common.Run([sys.executable, CYGTAR, '-C',
       os.path.join(OUT_DIR, sdktoolsdir), '-czf', tarname] + files_to_tar,
       cwd=NACL_DIR)
   sys.stdout.write('\n')
 
 
 def main(args):
   parser = optparse.OptionParser()
   parser.add_option('--pnacl', help='Enable pnacl build.',
       action='store_true', dest='pnacl', default=False)
-  parser.add_option('--examples', help='Rebuild the examples.',
-      action='store_true', dest='examples', default=False)
-  parser.add_option('--update', help='Rebuild the updater.',
-      action='store_true', dest='update', default=False)
+  parser.add_option('--examples', help='Only build the examples.',
+      action='store_true', dest='only_examples', default=False)
+  parser.add_option('--update', help='Only build the updater.',
+      action='store_true', dest='only_updater', default=False)
   parser.add_option('--skip-tar', help='Skip generating a tarball.',
       action='store_true', dest='skip_tar', default=False)
   parser.add_option('--archive', help='Force the archive step.',
       action='store_true', dest='archive', default=False)
   parser.add_option('--release', help='PPAPI release version.',
       dest='release', default=None)
-  
+
   options, args = parser.parse_args(args[1:])
   platform = getos.GetPlatform()
   arch = 'x86'
 
   builder_name = os.getenv('BUILDBOT_BUILDERNAME','')
   if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0:
     options.pnacl = True
 
   if options.pnacl:
     toolchains = ['pnacl']
   else:
     toolchains = ['newlib', 'glibc']
   print 'Building: ' + ' '.join(toolchains)
-  skip = options.examples or options.update
+  skip = options.only_examples or options.only_updater
 
-  skip_examples = skip
-  skip_update = skip
+  skip_examples = skip and not options.only_examples
+  skip_update = skip and not options.only_updater
   skip_untar = skip
   skip_build = skip
+  skip_test_updater = skip
   skip_tar = skip or options.skip_tar
 
-
-  if options.examples: skip_examples = False
-  skip_update = not options.update
-
-  if options.archive and (options.examples or options.skip_tar):
+  if options.archive and (options.only_examples or options.skip_tar):
     parser.error('Incompatible arguments with archive.')
 
   pepper_ver = str(int(build_utils.ChromeMajorVersion()))
   clnumber = lastchange.FetchVersionInfo(None).revision
   if options.release:
     pepper_ver = options.release
   print 'Building PEPPER %s at %s' % (pepper_ver, clnumber)
 
   if not skip_build:
     buildbot_common.BuildStep('Rerun hooks to get toolchains')
     buildbot_common.Run(['gclient', 'runhooks'],
                         cwd=SRC_DIR, shell=(platform=='win'))
 
-  buildbot_common.BuildStep('Clean Pepper Dir')
   pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
   if not skip_untar:
+    buildbot_common.BuildStep('Clean Pepper Dir')
     buildbot_common.RemoveDir(pepperdir)
     buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain'))
     buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
 
-  buildbot_common.BuildStep('Add Text Files')
-  files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
-  files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
-  oshelpers.Copy(['-v'] + files + [pepperdir])
+  if not skip_build:
+    buildbot_common.BuildStep('Add Text Files')
+    files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
+    files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
+    oshelpers.Copy(['-v'] + files + [pepperdir])
 
 
   # Clean out the temporary toolchain untar directory
   if not skip_untar:
     UntarToolchains(pepperdir, platform, arch, toolchains)
 
   if not skip_build:
     BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
 
-  buildbot_common.BuildStep('Copy make OS helpers')
-  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
-      os.path.join(pepperdir, 'tools'))
-  if platform == 'win':
-    buildbot_common.BuildStep('Add MAKE')
-    http_download.HttpDownload(GSTORE + MAKE,
-                             os.path.join(pepperdir, 'tools' ,'make.exe'))
+  if not skip_build:
+    buildbot_common.BuildStep('Copy make OS helpers')
+    buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
+        os.path.join(pepperdir, 'tools'))
+    if platform == 'win':
+      buildbot_common.BuildStep('Add MAKE')
+      http_download.HttpDownload(GSTORE + MAKE,
+                                 os.path.join(pepperdir, 'tools' ,'make.exe'))
 
   if not skip_examples:
     CopyExamples(pepperdir, toolchains)
 
+  tarname = 'naclsdk_' + platform + '.bz2'
+  if 'pnacl' in toolchains:
+    tarname = 'p' + tarname
+  tarfile = os.path.join(OUT_DIR, tarname)
+
   if not skip_tar:
     buildbot_common.BuildStep('Tar Pepper Bundle')
-    tarname = 'naclsdk_' + platform + '.bz2'
-    if 'pnacl' in toolchains:
-      tarname = 'p' + tarname
-    tarfile = os.path.join(OUT_DIR, tarname)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
          'pepper_' + pepper_ver], cwd=NACL_DIR)
 
-  # Archive on non-trybots.
-  if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
-    buildbot_common.BuildStep('Archive build')
-    buildbot_common.Archive(tarname,
-        'nativeclient-mirror/nacl/nacl_sdk/%s' % build_utils.ChromeVersion(),
-        OUT_DIR)
+  # build sdk update
+  if not skip_update:
+    BuildUpdater()
 
+  # start local server sharing a manifest + the new bundle
+  if not skip_test_updater:
+    try:
+      buildbot_common.BuildStep('Move bundle to localserver dir')
+      buildbot_common.MakeDir(SERVER_DIR)
+      buildbot_common.Move(tarfile, SERVER_DIR)
+      tarfile = os.path.join(SERVER_DIR, tarname)
+
+      buildbot_common.BuildStep('Run local server')
+      server = LocalHTTPServer(SERVER_DIR)
+
+      buildbot_common.BuildStep('Generate manifest')
+      with open(tarfile, 'rb') as tarfile_stream:
+        archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
+            tarfile_stream)
+      archive = manifest_util.Archive(manifest_util.GetHostOS())
+      archive.CopyFrom({'url': server.GetURL(tarname),
+                        'size': archive_size,
+                        'checksum': {'sha1': archive_sha1}})
+      bundle = manifest_util.Bundle('pepper_' + pepper_ver)
+      bundle.CopyFrom({
+          'revision': clnumber,
+          'repath': 'pepper_' + pepper_ver,
+          'version': pepper_ver,
+          'description': 'Chrome %s bundle, revision %s' % (
+              pepper_ver, clnumber),
+          'stability': 'dev',
+          'recommended': 'no',
+          'archives': [archive]})
+      manifest = manifest_util.SDKManifest()
+      manifest.SetBundle(bundle)
+      manifest_name = 'naclsdk_manifest2.json'
+      with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \
+          manifest_stream:
+        manifest_stream.write(manifest.GetManifestString())
+
+      # use newly built sdk updater to pull this bundle
+      buildbot_common.BuildStep('Update from local server')
+      updater_py = os.path.join(OUT_DIR, 'nacl_sdk', 'sdk_tools',
+          'sdk_update.py')
+      buildbot_common.Run([sys.executable, updater_py, '-U',
+          server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver])
+
+      # If we are testing examples, do it in the newly pulled directory.
+      pepperdir = os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver)
+
+    # kill server
+    finally:
+      server.Shutdown()
+
+  # build examples.
   if not skip_examples:
     buildbot_common.BuildStep('Test Build Examples')
     filelist = os.listdir(os.path.join(pepperdir, 'examples'))
     for filenode in filelist:
       dirnode = os.path.join(pepperdir, 'examples', filenode)
       makefile = os.path.join(dirnode, 'Makefile')
       if os.path.isfile(makefile):
         print "\n\nMake: " + dirnode
         buildbot_common.Run(['make', 'all', '-j8'],
                             cwd=os.path.abspath(dirnode), shell=True)
 
-  # Build SDK Tools
-  if not skip_update:
-    BuildUpdater()
+  # Archive on non-trybots.
+  if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
+    buildbot_common.BuildStep('Archive build')
+    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \
+        build_utils.ChromeVersion()
+    buildbot_common.Archive(tarname, bucket_path, OUT_DIR)
+
+    # generate "manifest snippet" for this archive.
+    if not skip_test_updater:
+      archive = bundle.GetArchive(manifest_util.GetHostOS())
+      archive.url = 'https://commondatastorage.googleapis.com/' \
+          'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % (
+              build_utils.ChromeVersion(), tarname)
+      manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
+      with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
+        manifest_snippet_stream.write(bundle.ToJSON())
+
+      buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
+                              step_link=False)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))