[NaCl SDK] Change build script to test sdk_updater, also push manifest snippets per bot.

native_client_sdk/src/build_tools/build_sdk.py

Index: native_client_sdk/src/build_tools/build_sdk.py
diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py
index a0df2a129a2a30251d990bc6851e6a35621898a9..62b77b29dedea94b8c082c0f1927c9ca25ca8a6f 100755
--- a/native_client_sdk/src/build_tools/build_sdk.py
+++ b/native_client_sdk/src/build_tools/build_sdk.py
@@ -17,15 +17,18 @@ and whether it should upload an SDK to file storage (GSTORE)
 
 
 # std python includes
+import multiprocessing
 import optparse
 import os
 import platform
+import subprocess
 import sys
 
 # local includes
 import buildbot_common
 import build_utils
 import lastchange
+import manifest_util
 
 # Create the various paths of interest
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -36,6 +39,7 @@ SRC_DIR = os.path.dirname(SDK_DIR)
 NACL_DIR = os.path.join(SRC_DIR, 'native_client')
 OUT_DIR = os.path.join(SRC_DIR, 'out')
 PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
+SERVER_DIR = os.path.join(OUT_DIR, 'local_server')
 
 
 # Add SDK make tools scripts to the python path.
@@ -51,17 +55,76 @@ MAKE = 'nacl_sdk/make_3_81/make.exe'
 CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')
 
 
+def HTTPServerProcess(conn, serve_dir):
+  """Run a local httpserver with a randomly-chosen port.
+
+  This function assumes it is run as a child process using multiprocessing.
+
+  Args:
+    conn: A connection to the parent process. The child process sends
+        the local port, and waits for a message from the parent to
+        stop serving.
+    serve_dir: The directory to serve. All files are accessible through
+       http://localhost:<port>/path/to/filename.
+  """
+  import BaseHTTPServer
+  import SimpleHTTPServer
+
+  os.chdir(serve_dir)
+  httpd = BaseHTTPServer.HTTPServer(('', 0),
+                                    SimpleHTTPServer.SimpleHTTPRequestHandler)
+  conn.send(httpd.server_address[1])  # the chosen port number
+  httpd.timeout = 0.5  # seconds
+  running = True
+  while running:
+    httpd.handle_request()
+    if conn.poll():
+      running = conn.recv()
+  conn.close()
+
+
+class LocalHTTPServer(object):
+  """Class to start a local HTTP server as a child process."""
+
+  def __init__(self, serve_dir):
+    parent_conn, child_conn = multiprocessing.Pipe()
+    self.process = multiprocessing.Process(target=HTTPServerProcess,
+                                           args=(child_conn, serve_dir))
+    self.process.start()
+    if parent_conn.poll(1):  # wait a second
+      self.port = parent_conn.recv()
+    else:
+      raise Exception('Unable to launch HTTP server.')
+
+    self.conn = parent_conn
+
+  def Shutdown(self):
+    """Send a message to the child HTTP server process and wait for it to
+        finish."""
+    self.conn.send(False)
+    self.process.join()
+
+  def GetURL(self, rel_url):
+    """Get the full url for a file on the local HTTP server.
+
+    Args:
+      rel_url: A URL fragment to convert to a full URL. For example,
+          GetURL('foobar.baz') -> 'http://localhost:1234/foobar.baz'
+    """
+    return 'http://localhost:%d/%s' % (self.port, rel_url)
+
+
 def AddMakeBat(pepperdir, makepath):
   """Create a simple batch file to execute Make.
-  
+
   Creates a simple batch file named make.bat for the Windows platform at the
   given path, pointing to the Make executable in the SDK."""
-  
+
   makepath = os.path.abspath(makepath)
   if not makepath.startswith(pepperdir):
     buildbot_common.ErrorExit('Make.bat not relative to Pepper directory: ' +
                               makepath)
-  
+
   makeexe = os.path.abspath(os.path.join(pepperdir, 'tools'))
   relpath = os.path.relpath(makeexe, makepath)
 
@@ -202,7 +265,7 @@ def InstallHeaders(tc_dst_inc, pepper_ver, tc_name):
   buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'c', 'dev', '*.h'),
           os.path.join(ppapi, 'c', 'dev'))
 
-  # Run the generator to overwrite IDL files  
+  # Run the generator to overwrite IDL files
   buildbot_common.Run([sys.executable, 'generator.py', '--wnone', '--cgen',
        '--release=M' + pepper_ver, '--verbose', '--dstroot=%s/c' % ppapi],
        cwd=os.path.join(PPAPI_DIR, 'generators'))
@@ -259,7 +322,7 @@ def UntarToolchains(pepperdir, platform, arch, toolchains):
     tarfile = GetNewlibToolchain(platform, arch)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                         cwd=NACL_DIR)
-  
+
     # Then rename/move it to the pepper toolchain directory
     srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk')
     newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
@@ -270,7 +333,7 @@ def UntarToolchains(pepperdir, platform, arch, toolchains):
     tarfile = GetGlibcToolchain(platform, arch)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                         cwd=NACL_DIR)
-  
+
     # Then rename/move it to the pepper toolchain directory
     srcdir = os.path.join(tmpdir, 'toolchain', tcname)
     glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
@@ -349,7 +412,7 @@ EXAMPLE_MAP = {
     'fullscreen_tumbler',
     'gamepad',
     'geturl',
-    'hello_world_interactive', 
+    'hello_world_interactive',
     'hello_world_newlib',
     'input_events',
     'load_progress',
@@ -458,17 +521,17 @@ def main(args):
   parser = optparse.OptionParser()
   parser.add_option('--pnacl', help='Enable pnacl build.',
       action='store_true', dest='pnacl', default=False)
-  parser.add_option('--examples', help='Rebuild the examples.',
-      action='store_true', dest='examples', default=False)
-  parser.add_option('--update', help='Rebuild the updater.',
-      action='store_true', dest='update', default=False)
+  parser.add_option('--examples', help='Only build the examples.',
+      action='store_true', dest='only_examples', default=False)
+  parser.add_option('--update', help='Only build the updater.',
+      action='store_true', dest='only_updater', default=False)
   parser.add_option('--skip-tar', help='Skip generating a tarball.',
       action='store_true', dest='skip_tar', default=False)
   parser.add_option('--archive', help='Force the archive step.',
       action='store_true', dest='archive', default=False)
   parser.add_option('--release', help='PPAPI release version.',
       dest='release', default=None)
-  
+
   options, args = parser.parse_args(args[1:])
   platform = getos.GetPlatform()
   arch = 'x86'
@@ -482,19 +545,16 @@ def main(args):
   else:
     toolchains = ['newlib', 'glibc']
   print 'Building: ' + ' '.join(toolchains)
-  skip = options.examples or options.update
+  skip = options.only_examples or options.only_updater
 
-  skip_examples = skip
-  skip_update = skip
+  skip_examples = skip and not options.only_examples
+  skip_update = skip and not options.only_updater
   skip_untar = skip
   skip_build = skip
+  skip_test_updater = skip
   skip_tar = skip or options.skip_tar
 
-
-  if options.examples: skip_examples = False
-  skip_update = not options.update
-
-  if options.archive and (options.examples or options.skip_tar):
+  if options.archive and (options.only_examples or options.skip_tar):
     parser.error('Incompatible arguments with archive.')
 
   pepper_ver = str(int(build_utils.ChromeMajorVersion()))
@@ -508,17 +568,18 @@ def main(args):
     buildbot_common.Run(['gclient', 'runhooks'],
                         cwd=SRC_DIR, shell=(platform=='win'))
 
-  buildbot_common.BuildStep('Clean Pepper Dir')
   pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
   if not skip_untar:
+    buildbot_common.BuildStep('Clean Pepper Dir')
     buildbot_common.RemoveDir(pepperdir)
     buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain'))
     buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
 
-  buildbot_common.BuildStep('Add Text Files')
-  files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
-  files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
-  oshelpers.Copy(['-v'] + files + [pepperdir])
+  if not skip_build:
+    buildbot_common.BuildStep('Add Text Files')
+    files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
+    files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
+    oshelpers.Copy(['-v'] + files + [pepperdir])
 
 
   # Clean out the temporary toolchain untar directory
@@ -528,33 +589,83 @@ def main(args):
   if not skip_build:
     BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
 
-  buildbot_common.BuildStep('Copy make OS helpers')
-  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
-      os.path.join(pepperdir, 'tools'))
-  if platform == 'win':
-    buildbot_common.BuildStep('Add MAKE')
-    http_download.HttpDownload(GSTORE + MAKE,
-                             os.path.join(pepperdir, 'tools' ,'make.exe'))
+  if not skip_build:
+    buildbot_common.BuildStep('Copy make OS helpers')
+    buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
+        os.path.join(pepperdir, 'tools'))
+    if platform == 'win':
+      buildbot_common.BuildStep('Add MAKE')
+      http_download.HttpDownload(GSTORE + MAKE,
+                                 os.path.join(pepperdir, 'tools' ,'make.exe'))
 
   if not skip_examples:
     CopyExamples(pepperdir, toolchains)
 
+  tarname = 'naclsdk_' + platform + '.bz2'
+  if 'pnacl' in toolchains:
+    tarname = 'p' + tarname
+  tarfile = os.path.join(OUT_DIR, tarname)
+
   if not skip_tar:
     buildbot_common.BuildStep('Tar Pepper Bundle')
-    tarname = 'naclsdk_' + platform + '.bz2'
-    if 'pnacl' in toolchains:
-      tarname = 'p' + tarname
-    tarfile = os.path.join(OUT_DIR, tarname)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
          'pepper_' + pepper_ver], cwd=NACL_DIR)
 
-  # Archive on non-trybots.
-  if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
-    buildbot_common.BuildStep('Archive build')
-    buildbot_common.Archive(tarname,
-        'nativeclient-mirror/nacl/nacl_sdk/%s' % build_utils.ChromeVersion(),
-        OUT_DIR)
+  # build sdk update
+  if not skip_update:
+    BuildUpdater()
 
+  # start local server sharing a manifest + the new bundle
+  if not skip_test_updater:
+    try:
+      buildbot_common.BuildStep('Move bundle to localserver dir')
+      buildbot_common.MakeDir(SERVER_DIR)
+      buildbot_common.Move(tarfile, SERVER_DIR)
+      tarfile = os.path.join(SERVER_DIR, tarname)
+
+      buildbot_common.BuildStep('Run local server')
+      server = LocalHTTPServer(SERVER_DIR)
+
+      buildbot_common.BuildStep('Generate manifest')
+      with open(tarfile, 'rb') as tarfile_stream:
+        archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
+            tarfile_stream)
+      archive = manifest_util.Archive(manifest_util.GetHostOS())
+      archive.CopyFrom({'url': server.GetURL(tarname),
+                        'size': archive_size,
+                        'checksum': {'sha1': archive_sha1}})
+      bundle = manifest_util.Bundle('pepper_' + pepper_ver)
+      bundle.CopyFrom({
+          'revision': clnumber,
+          'repath': 'pepper_' + pepper_ver,
+          'version': pepper_ver,
+          'description': 'Chrome %s bundle, revision %s' % (
+              pepper_ver, clnumber),
+          'stability': 'dev',
+          'recommended': 'no',
+          'archives': [archive]})
+      manifest = manifest_util.SDKManifest()
+      manifest.SetBundle(bundle)
+      manifest_name = 'naclsdk_manifest2.json'
+      with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \
+          manifest_stream:
+        manifest_stream.write(manifest.GetManifestString())
+
+      # use newly built sdk updater to pull this bundle
+      buildbot_common.BuildStep('Update from local server')
+      updater_py = os.path.join(OUT_DIR, 'nacl_sdk', 'sdk_tools',
+          'sdk_update.py')
+      buildbot_common.Run([sys.executable, updater_py, '-U',
+          server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver])
+
+      # If we are testing examples, do it in the newly pulled directory.
+      pepperdir = os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver)
+
+    # kill server
+    finally:
+      server.Shutdown()
+
+  # build examples.
   if not skip_examples:
     buildbot_common.BuildStep('Test Build Examples')
     filelist = os.listdir(os.path.join(pepperdir, 'examples'))
@@ -566,9 +677,25 @@ def main(args):
         buildbot_common.Run(['make', 'all', '-j8'],
                             cwd=os.path.abspath(dirnode), shell=True)
 
-  # Build SDK Tools
-  if not skip_update:
-    BuildUpdater()
+  # Archive on non-trybots.
+  if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
+    buildbot_common.BuildStep('Archive build')
+    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \
+        build_utils.ChromeVersion()
+    buildbot_common.Archive(tarname, bucket_path, OUT_DIR)
+
+    # generate "manifest snippet" for this archive.
+    if not skip_test_updater:
+      archive = bundle.GetArchive(manifest_util.GetHostOS())
+      archive.url = 'https://commondatastorage.googleapis.com/' \
+          'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % (
+              build_utils.ChromeVersion(), tarname)
+      manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
+      with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
+        manifest_snippet_stream.write(bundle.ToJSON())
+
+      buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
+                              step_link=False)
 
   return 0