| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 '''Utility to update the SDK manifest file in the build_tools directory''' | |
| 7 | |
| 8 | |
| 9 import optparse | |
| 10 import os | |
| 11 import re | |
| 12 import string | |
| 13 import subprocess | |
| 14 import sys | |
| 15 import urllib2 | |
| 16 | |
| 17 # Create the various paths of interest | |
| 18 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| 19 SDK_SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR)) | |
| 20 SRC_DIR = os.path.dirname(os.path.dirname(SDK_SRC_DIR)) | |
| 21 NACL_DIR = os.path.join(SRC_DIR, 'native_client') | |
| 22 | |
| 23 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) | |
| 24 sys.path.append(os.path.join(NACL_DIR, 'build')) | |
| 25 | |
| 26 import sdk_update | |
| 27 | |
| 28 HELP = '''"Usage: %prog [-b bundle] [options]" | |
| 29 | |
| 30 Actions for particular bundles: | |
| 31 sdk_tools: Upload the most recently built nacl_sdk.zip and sdk_tools.tgz | |
| 32 files to the server and update the manifest file | |
| 33 pepper_??: Download the latest pepper builds off the appropriate branch, | |
| 34 upload these files to the appropriate location on the server, and | |
| 35 update the manifest file. | |
| 36 <others>: Only update manifest file -- you'll need to upload the file yourself | |
| 37 ''' | |
| 38 | |
| 39 # Map option keys to manifest attribute key. Option keys are used to retrieve | |
| 40 # option values from cmd-line options. Manifest attribute keys label the | |
| 41 # corresponding value in the manifest object. | |
| 42 OPTION_KEY_MAP = { | |
| 43 # option key manifest attribute key | |
| 44 'bundle_desc_url': 'desc_url', | |
| 45 'bundle_revision': sdk_update.REVISION_KEY, | |
| 46 'bundle_version': sdk_update.VERSION_KEY, | |
| 47 'desc': 'description', | |
| 48 'recommended': 'recommended', | |
| 49 'stability': 'stability', | |
| 50 } | |
| 51 # Map options keys to platform key, as stored in the bundle. | |
| 52 OPTION_KEY_TO_PLATFORM_MAP = { | |
| 53 'mac_arch_url': 'mac', | |
| 54 'win_arch_url': 'win', | |
| 55 'linux_arch_url': 'linux', | |
| 56 'all_arch_url': 'all', | |
| 57 } | |
| 58 | |
| 59 NACL_SDK_ROOT = os.path.dirname(os.path.dirname(os.path.dirname( | |
| 60 os.path.abspath(__file__)))) | |
| 61 | |
| 62 BUILD_TOOLS_OUT = os.path.join(NACL_SDK_ROOT, 'scons-out', 'build', 'obj', | |
| 63 'build_tools') | |
| 64 | |
| 65 BUNDLE_SDK_TOOLS = 'sdk_tools' | |
| 66 BUNDLE_PEPPER_MATCHER = re.compile('^pepper_([0-9]+)$') | |
| 67 IGNORE_OPTIONS = set([ | |
| 68 'archive_id', 'gsutil', 'manifest_file', 'upload', 'root_url']) | |
| 69 | |
| 70 | |
| 71 class Error(Exception): | |
| 72 '''Generic error/exception for update_manifest module''' | |
| 73 pass | |
| 74 | |
| 75 | |
| 76 def UpdateBundle(bundle, options): | |
| 77 ''' Update the bundle per content of the options. | |
| 78 | |
| 79 Args: | |
| 80 options: options data. Attributes that are used are also deleted from | |
| 81 options.''' | |
| 82 # Check, set and consume individual bundle options. | |
| 83 for option_key, attribute_key in OPTION_KEY_MAP.iteritems(): | |
| 84 option_val = getattr(options, option_key, None) | |
| 85 if option_val is not None: | |
| 86 bundle[attribute_key] = option_val | |
| 87 delattr(options, option_key) | |
| 88 # Validate what we have so far; we may just avoid going through a lengthy | |
| 89 # download, just to realize that some other trivial stuff is missing. | |
| 90 bundle.Validate() | |
| 91 # Check and consume archive-url options. | |
| 92 for option_key, host_os in OPTION_KEY_TO_PLATFORM_MAP.iteritems(): | |
| 93 platform_url = getattr(options, option_key, None) | |
| 94 if platform_url is not None: | |
| 95 bundle.UpdateArchive(host_os, platform_url) | |
| 96 delattr(options, option_key) | |
| 97 | |
| 98 | |
| 99 class UpdateSDKManifest(sdk_update.SDKManifest): | |
| 100 '''Adds functions to SDKManifest that are only used in update_manifest''' | |
| 101 | |
| 102 def _ValidateBundleName(self, name): | |
| 103 ''' Verify that name is a valid bundle. | |
| 104 | |
| 105 Args: | |
| 106 name: the proposed name for the bundle. | |
| 107 | |
| 108 Return: | |
| 109 True if the name is valid for a bundle, False otherwise.''' | |
| 110 valid_char_set = '()-_.%s%s' % (string.ascii_letters, string.digits) | |
| 111 name_len = len(name) | |
| 112 return (name_len > 0 and all(c in valid_char_set for c in name)) | |
| 113 | |
| 114 def _UpdateManifestVersion(self, options): | |
| 115 ''' Update the manifest version number from the options | |
| 116 | |
| 117 Args: | |
| 118 options: options data containing an attribute self.manifest_version ''' | |
| 119 version_num = int(options.manifest_version) | |
| 120 self._manifest_data['manifest_version'] = version_num | |
| 121 del options.manifest_version | |
| 122 | |
| 123 def _UpdateBundle(self, options): | |
| 124 ''' Update or setup a bundle from the options. | |
| 125 | |
| 126 Args: | |
| 127 options: options data containing at least a valid bundle_name | |
| 128 attribute. Other relevant bundle attributes will also be | |
| 129 used (and consumed) by this function. ''' | |
| 130 # Get and validate the bundle name | |
| 131 if not self._ValidateBundleName(options.bundle_name): | |
| 132 raise Error('Invalid bundle name: "%s"' % options.bundle_name) | |
| 133 bundle_name = options.bundle_name | |
| 134 del options.bundle_name | |
| 135 # Get the corresponding bundle, or create it. | |
| 136 bundle = self.GetBundle(bundle_name) | |
| 137 if not bundle: | |
| 138 bundle = sdk_update.Bundle(bundle_name) | |
| 139 self.SetBundle(bundle) | |
| 140 UpdateBundle(bundle, options) | |
| 141 | |
| 142 def _VerifyAllOptionsConsumed(self, options, bundle_name): | |
| 143 ''' Verify that all the options have been used. Raise an exception if | |
| 144 any valid option has not been used. Returns True if all options have | |
| 145 been consumed. | |
| 146 | |
| 147 Args: | |
| 148 options: the object containing the remaining unused options attributes. | |
| 149 bundle_name: The name of the bundle, or None if it's missing.''' | |
| 150 # Any option left in the list should have value = None | |
| 151 for key, val in options.__dict__.items(): | |
| 152 if val != None and key not in IGNORE_OPTIONS: | |
| 153 if bundle_name: | |
| 154 raise Error('Unused option "%s" for bundle "%s"' % (key, bundle_name)) | |
| 155 else: | |
| 156 raise Error('No bundle name specified') | |
| 157 return True | |
| 158 | |
| 159 def UpdateManifest(self, options): | |
| 160 ''' Update the manifest object with values from the command-line options | |
| 161 | |
| 162 Args: | |
| 163 options: options object containing attribute for the command-line options. | |
| 164 Note that all the non-trivial options are consumed. | |
| 165 ''' | |
| 166 # Go over all the options and update the manifest data accordingly. | |
| 167 # Valid options are consumed as they are used. This gives us a way to | |
| 168 # verify that all the options are used. | |
| 169 if options.manifest_version is not None: | |
| 170 self._UpdateManifestVersion(options) | |
| 171 # Keep a copy of bundle_name, which will be consumed by UpdateBundle, for | |
| 172 # use in _VerifyAllOptionsConsumed below. | |
| 173 bundle_name = options.bundle_name | |
| 174 if bundle_name is not None: | |
| 175 self._UpdateBundle(options) | |
| 176 self._VerifyAllOptionsConsumed(options, bundle_name) | |
| 177 self._ValidateManifest() | |
| 178 | |
| 179 def ValidateManifestLinks(self): | |
| 180 '''Validates all the links in the manifest file and throws if one is bad''' | |
| 181 valid = True | |
| 182 for bundle in self._manifest_data[sdk_update.BUNDLES_KEY]: | |
| 183 for archive in bundle.GetArchives(): | |
| 184 stream = None | |
| 185 try: | |
| 186 print "Checking size of data at link: %s" % archive.GetUrl() | |
| 187 stream = urllib2.urlopen(archive.GetUrl()) | |
| 188 server_size = int(stream.info()[sdk_update.HTTP_CONTENT_LENGTH]) | |
| 189 if server_size != archive.GetSize(): | |
| 190 sys.stderr.write('Size mismatch for %s. Expected %s but got %s\n' % | |
| 191 (archive.GetUrl(), archive.GetSize(), server_size)) | |
| 192 sys.stderr.flush() | |
| 193 valid = False | |
| 194 finally: | |
| 195 if stream: | |
| 196 stream.close() | |
| 197 if not valid: | |
| 198 raise Error('Files on server do not match the manifest file') | |
| 199 | |
| 200 | |
| 201 class GsUtil(object): | |
| 202 def __init__(self, gsutil): | |
| 203 '''gsutil is the path to the gsutil executable''' | |
| 204 self.gsutil = gsutil | |
| 205 self.root = 'gs://nativeclient-mirror/nacl/nacl_sdk' | |
| 206 | |
| 207 def GetURI(self, path): | |
| 208 '''Return the full gs:// URI for a given relative path''' | |
| 209 return '/'.join([self.root, path]) | |
| 210 | |
| 211 def Run(self, command): | |
| 212 '''Runs gsutil with a given argument list and returns exit status''' | |
| 213 args = [self.gsutil] + command | |
| 214 print 'GSUtil.Run(%s)' % args | |
| 215 sys.stdout.flush() | |
| 216 return subprocess.call(args) | |
| 217 | |
| 218 def CheckIfExists(self, path): | |
| 219 '''Check whether a given path exists on commondatastorage | |
| 220 | |
| 221 Args: | |
| 222 path: path relative to SDK root directory on the server | |
| 223 | |
| 224 Returns: True if it exists, False if it does not''' | |
| 225 # todo(mball): Be a little more intelligent about this check and compare | |
| 226 # the output strings against expected values | |
| 227 return self.Run(['ls', self.GetURI(path)]) == 0 | |
| 228 | |
| 229 def Copy(self, source, destination): | |
| 230 '''Copies a given source file to a destination path and makes it readable | |
| 231 | |
| 232 Args: | |
| 233 source: path to source file on local filesystem | |
| 234 destination: path to destination, relative to root directory''' | |
| 235 args = ['cp', '-a', 'public-read', source, self.GetURI(destination)] | |
| 236 if self.Run(args) != 0: | |
| 237 raise Error('Unable to copy %s to %s' % (source, destination)) | |
| 238 | |
| 239 | |
| 240 class UpdateSDKManifestFile(sdk_update.SDKManifestFile): | |
| 241 '''Adds functions to SDKManifestFile that are only used in update_manifest''' | |
| 242 | |
| 243 def __init__(self, options): | |
| 244 '''Create a new SDKManifest object with default contents. | |
| 245 | |
| 246 If |json_filepath| is specified, and it exists, its contents are loaded and | |
| 247 used to initialize the internal manifest. | |
| 248 | |
| 249 Args: | |
| 250 json_filepath: path to json file to read/write, or None to write a new | |
| 251 manifest file to stdout. | |
| 252 ''' | |
| 253 # Strip-off all the I/O-based options that do not relate to bundles | |
| 254 self._json_filepath = options.manifest_file | |
| 255 self.gsutil = GsUtil(options.gsutil) | |
| 256 self.options = options | |
| 257 self._manifest = UpdateSDKManifest() | |
| 258 if self._json_filepath: | |
| 259 self._LoadFile() | |
| 260 | |
| 261 def _HandleSDKTools(self): | |
| 262 '''Handles the sdk_tools bundle''' | |
| 263 # General sanity checking of parameters | |
| 264 SDK_TOOLS_FILES = ['sdk_tools.tgz', 'nacl_sdk.zip'] | |
| 265 options = self.options | |
| 266 if options.bundle_version is None: | |
| 267 options.bundle_version = sdk_update.MAJOR_REV | |
| 268 if options.bundle_version != sdk_update.MAJOR_REV: | |
| 269 raise Error('Specified version (%s) does not match MAJOR_REV (%s)' % | |
| 270 (options.bundle_version, sdk_update.MAJOR_REV)) | |
| 271 if options.bundle_revision is None: | |
| 272 options.bundle_revision = sdk_update.MINOR_REV | |
| 273 if options.bundle_revision != sdk_update.MINOR_REV: | |
| 274 raise Error('Specified revision (%s) does not match MINOR_REV (%s)' % | |
| 275 (options.bundle_revision, sdk_update.MINOR_REV)) | |
| 276 version = '%s.%s' % (options.bundle_version, options.bundle_revision) | |
| 277 # Update the remaining options | |
| 278 if options.desc is None: | |
| 279 options.desc = ('Native Client SDK Tools, revision %s.%s' % | |
| 280 (options.bundle_version, options.bundle_revision)) | |
| 281 options.recommended = options.recommended or 'yes' | |
| 282 options.stability = options.stability or 'stable' | |
| 283 if options.upload: | |
| 284 # Check whether the tools already exist | |
| 285 for name in SDK_TOOLS_FILES: | |
| 286 path = '/'.join([version, name]) | |
| 287 if self.gsutil.CheckIfExists(path): | |
| 288 raise Error('File already exists at %s' % path) | |
| 289 # Upload the tools files to the server | |
| 290 for name in SDK_TOOLS_FILES: | |
| 291 source = os.path.join(BUILD_TOOLS_OUT, name) | |
| 292 destination = '/'.join([version, name]) | |
| 293 self.gsutil.Copy(source, destination) | |
| 294 url = '/'.join([options.root_url, version, 'sdk_tools.tgz']) | |
| 295 options.mac_arch_url = options.mac_arch_url or url | |
| 296 options.linux_arch_url = options.linux_arch_url or url | |
| 297 options.win_arch_url = options.win_arch_url or url | |
| 298 | |
| 299 def _HandlePepper(self): | |
| 300 '''Handles the pepper bundles''' | |
| 301 options = self.options | |
| 302 match = BUNDLE_PEPPER_MATCHER.match(options.bundle_name) | |
| 303 if match is not None: | |
| 304 options.bundle_version = int(match.group(1)) | |
| 305 if options.bundle_version is None: | |
| 306 raise Error('Need to specify a bundle version') | |
| 307 if options.bundle_revision is None: | |
| 308 raise Error('Need to specify a bundle revision') | |
| 309 if options.bundle_name == 'pepper': | |
| 310 self.options.bundle_name = 'pepper_%s' % options.bundle_version | |
| 311 if options.desc is None: | |
| 312 options.desc = ('Chrome %s bundle, revision %s' % | |
| 313 (options.bundle_version, options.bundle_revision)) | |
| 314 root_url = options.root_url | |
| 315 if options.archive_id: | |
| 316 # Support archive names like trunk.113440 or 17.0.963.3, which is how | |
| 317 # the Chrome builders archive things. | |
| 318 root_url = '/'.join([root_url, options.archive_id]) | |
| 319 else: | |
| 320 # This is the old archive naming scheme | |
| 321 root_url = '%s/pepper_%s_%s' % (root_url, options.bundle_version, | |
| 322 options.bundle_revision) | |
| 323 options.mac_arch_url = '/'.join([root_url, 'naclsdk_mac.bz2']) | |
| 324 options.linux_arch_url = '/'.join([root_url, 'naclsdk_linux.bz2']) | |
| 325 options.win_arch_url = '/'.join([root_url, 'naclsdk_win.bz2']) | |
| 326 | |
| 327 def HandleBundles(self): | |
| 328 '''Handles known bundles by automatically uploading files''' | |
| 329 bundle_name = self.options.bundle_name | |
| 330 print 'bundle_name=' + bundle_name | |
| 331 if bundle_name == BUNDLE_SDK_TOOLS: | |
| 332 self._HandleSDKTools() | |
| 333 elif bundle_name.startswith('pepper'): | |
| 334 self._HandlePepper() | |
| 335 | |
| 336 def UpdateWithOptions(self): | |
| 337 ''' Update the manifest file with the given options. Create the manifest | |
| 338 if it doesn't already exists. Raises an Error if the manifest doesn't | |
| 339 validate after updating. | |
| 340 | |
| 341 Args: | |
| 342 options: option data''' | |
| 343 # UpdateManifest does not know how to deal with file-related options | |
| 344 self._manifest.UpdateManifest(self.options) | |
| 345 self.WriteFile() | |
| 346 | |
| 347 | |
| 348 def CommandPush(options, args, manifest_file): | |
| 349 '''Check the manifest file and push it to the server if it's okay''' | |
| 350 print 'Running Push with options=%s and args=%s' % (options, args) | |
| 351 manifest = manifest_file._manifest | |
| 352 manifest.UpdateManifest(options) | |
| 353 print 'Validating links within manifest file' | |
| 354 manifest.ValidateManifestLinks() | |
| 355 print 'Copying manifest file to server' | |
| 356 manifest_file.gsutil.Copy(options.manifest_file, 'naclsdk_manifest.json') | |
| 357 | |
| 358 | |
| 359 def main(argv): | |
| 360 '''Main entry for update_manifest.py''' | |
| 361 | |
| 362 buildtools_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| 363 parser = optparse.OptionParser(usage=HELP) | |
| 364 | |
| 365 # Setup options | |
| 366 parser.add_option( | |
| 367 '-a', '--archive-id', dest='archive_id', | |
| 368 default=None, | |
| 369 help='Archive identifier, produced by the Chromium builders; string ' | |
| 370 'like "trunk.113440" or "17.0.963.3". Used with --root-url to ' | |
| 371 'build the full archive URL. If not set the archive id defaults to ' | |
| 372 '"pepper_<version>_<revision>"') | |
| 373 parser.add_option( | |
| 374 '-b', '--bundle-version', dest='bundle_version', | |
| 375 type='int', | |
| 376 default=None, | |
| 377 help='Required: Version number for the bundle.') | |
| 378 parser.add_option( | |
| 379 '-B', '--bundle-revision', dest='bundle_revision', | |
| 380 type='int', | |
| 381 default=None, | |
| 382 help='Required: Revision number for the bundle.') | |
| 383 parser.add_option( | |
| 384 '-d', '--description', dest='desc', | |
| 385 default=None, | |
| 386 help='Required: Description for this bundle.') | |
| 387 parser.add_option( | |
| 388 '-f', '--manifest-file', dest='manifest_file', | |
| 389 default=os.path.join(buildtools_dir, 'json', | |
| 390 sdk_update.MANIFEST_FILENAME), | |
| 391 help='location of manifest file to read and update') | |
| 392 parser.add_option( | |
| 393 '-g', '--gsutil', dest='gsutil', | |
| 394 default='gsutil', help='location of gsutil tool for uploading bundles') | |
| 395 parser.add_option( | |
| 396 '-L', '--linux-archive', dest='linux_arch_url', | |
| 397 default=None, | |
| 398 help='URL for the Linux archive.') | |
| 399 parser.add_option( | |
| 400 '-M', '--mac-archive', dest='mac_arch_url', | |
| 401 default=None, | |
| 402 help='URL for the Mac archive.') | |
| 403 parser.add_option( | |
| 404 '-n', '--bundle-name', dest='bundle_name', | |
| 405 default=None, | |
| 406 help='Required: Name of the bundle.') | |
| 407 parser.add_option( | |
| 408 '-r', '--recommended', dest='recommended', | |
| 409 choices=sdk_update.YES_NO_LITERALS, | |
| 410 default=None, | |
| 411 help='Required: whether this bundle is recommended. One of "yes" or "no"') | |
| 412 parser.add_option( | |
| 413 '-R', '--root-url', dest='root_url', | |
| 414 default='http://commondatastorage.googleapis.com/nativeclient-mirror/' | |
| 415 'nacl/nacl_sdk', | |
| 416 help='Root url for uploading') | |
| 417 parser.add_option( | |
| 418 '-s', '--stability', dest='stability', | |
| 419 choices=sdk_update.STABILITY_LITERALS, | |
| 420 default=None, | |
| 421 help='Required: Stability for this bundle; one of. ' | |
| 422 '"obsolete", "post_stable", "stable", "beta", "dev", "canary".') | |
| 423 parser.add_option( | |
| 424 '-u', '--desc-url', dest='bundle_desc_url', | |
| 425 default=None, | |
| 426 help='Optional: URL to follow to read additional bundle info.') | |
| 427 parser.add_option( | |
| 428 '-U', '--upload', dest='upload', default=False, action='store_true', | |
| 429 help='Indicates whether to upload bundle to server') | |
| 430 parser.add_option( | |
| 431 '-v', '--manifest-version', dest='manifest_version', | |
| 432 type='int', | |
| 433 default=None, | |
| 434 help='Required for new manifest files: ' | |
| 435 'Version number for the manifest.') | |
| 436 parser.add_option( | |
| 437 '-W', '--win-archive', dest='win_arch_url', | |
| 438 default=None, | |
| 439 help='URL for the Windows archive.') | |
| 440 | |
| 441 # Parse options and arguments and check. | |
| 442 (options, args) = parser.parse_args(argv) | |
| 443 manifest_file = UpdateSDKManifestFile(options) | |
| 444 if len(args) == 0: | |
| 445 manifest_file.HandleBundles() | |
| 446 manifest_file.UpdateWithOptions() | |
| 447 return 0 | |
| 448 | |
| 449 COMMANDS = { | |
| 450 'push': CommandPush | |
| 451 } | |
| 452 def CommandUnknown(options, args, manifest_file): | |
| 453 raise Error("Unknown command %s" % args[0]) | |
| 454 try: | |
| 455 COMMANDS.get(args[0], CommandUnknown)(options, args, manifest_file) | |
| 456 except Error as error: | |
| 457 print "Error: %s" % error | |
| 458 return 1 | |
| 459 return 0 | |
| 460 | |
| 461 | |
| 462 if __name__ == '__main__': | |
| 463 sys.exit(main(sys.argv[1:])) | |
| OLD | NEW |
|---|
Chromium Code Reviews
