Add CIPD GAE SDK recipe module and packager.

scripts/slave/recipe_modules/gae_sdk/api.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import re
+
+from recipe_engine import recipe_api
+
+
+class GaeSdkApi(recipe_api.RecipeApi):
+
+  PLAT_PYTHON = 'python'
+  PLAT_GO = 'go'
+
+  # The Google Storage GAE SDK bucket base. All SDK packages are stored in here
+  # under a basename + version ZIP file.
+  GS_BUCKET_BASE = 'gs://appengine-sdks/featured'
+  # The GS path to the "LATEST" YAML file.
+  GS_VERSION_YAML = '%s/VERSION' % (GS_BUCKET_BASE,)
+
+  # Hacky regex for the "release" YAML variable.
+  RE_RELEASE = re.compile(r'^release:\s+"([^"]+)"$')
+
+  # Map of {Platform => {Arch => (base, dirname)}}. Platform names are
+  # "recipe_engine/platform" values.
+  _PKG_MAP = {
+      PLAT_PYTHON: {
+        'all': ('google_appengine_', 'google_appengine'),
+      },
+      PLAT_GO: {
+        'linux-amd64': ('go_appengine_sdk_linux_amd64-', 'go_appengine'),
+        'linux-386': ('go_appengine_sdk_linux_386-', 'go_appengine'),
+        'mac-amd64': ('go_appengine_sdk_darwin_amd64-', 'go_appengine'),
+      },
+  }
+
+  # Map of architecture bitness to CIPD bitness suffix.
+  _BITS_MAP = {
+      64: 'amd64',
+      32: '386',

[iannucci, 2016/09/30 21:05:21]

hm... I didn't realize it was 386 instead of x86. That's an interesting choice.

[dnj, 2016/09/30 22:54:54]

I've generally seen i386 / i686 vs. amd64 / x86_64. I think one is GNU and the
other is Intel or something.


... like assembly!

+  }
+
+
+  class VersionParseError(Exception):
+    pass
+
+
+  class PackageNotFound(Exception):
+    def __init__(self, plat, arch):
+      super(GaeSdkApi.PackageNotFound, self).__init__(
+          'Package not found for %s on %s' % (plat, arch))
+      self.plat = plat
+      self.arch = arch
+
+
+  def __init__(self, *args, **kwargs):
+    super(GaeSdkApi, self).__init__(*args, **kwargs)
+    self._latest = None
+
+  def _step_name(self, desc):
+    return 'GAE SDK: %s' % (desc,)
+
+  @property
+  def latest_ref(self):
+    return 'latest'
+
+  def version_tag(self, version):
+    return ('gae_sdk_version', version)
+
+  def _package_spec(self, plat, arch):
+    plat_dict = self._PKG_MAP.get(plat, {})
+    for a in (arch, 'all'):
+      spec = plat_dict.get(a)
+      if spec:
+        download_base, dirname = spec
+        pkg_name = 'infra/gae_sdk/%s/%s' % (plat, a)
+        return pkg_name, download_base, dirname
+    raise self.PackageNotFound(plat, arch)
+
+  def package(self, plat, arch=None):
+    arch = arch or '%s-%s' % (
+        self.m.platform.name, self._BITS_MAP[self.m.platform.bits])
+    pkg_name, _, _ = self._package_spec(plat, arch)
+    return pkg_name
+
+  @property
+  def platforms(self):
+    return sorted(self._PKG_MAP.keys())
+
+  @property
+  def all_packages(self):
+    for plat, arch_dict in sorted(self._PKG_MAP.items()):
+      for arch in sorted(arch_dict.keys()):
+        yield plat, arch
+
+  def latest_upstream_version(self):
+    if self._latest is None:
+      step_result = self.m.gsutil.cat(
+          self.GS_VERSION_YAML,
+          name=self._step_name('Get Latest'),
+          stdout=self.m.raw_io.output())
+      self._latest = self._parse_latest_yaml(step_result.stdout)
+      step_result.presentation.step_text += ' %s' % (self._latest,)
+    return self._latest
+
+  @classmethod
+  def _parse_latest_yaml(cls, text):
+    # Rather than import a YAML parser, we will specifically search for the
+    # string:
+    #
+    # release: "<version>"
+    for line in text.splitlines():
+      m = cls.RE_RELEASE.match(line)
+      if m:
+        return m.group(1)
+    raise cls.VersionParseError('Could not parse release version from YAML.')
+
+  def download_and_unpack(self, plat, arch, version):

[iannucci, 2016/09/30 21:05:21]

can we move this to the packaging recipe? Otherwise users may be tempted to call
this directly.... I was initially confused and thought that this was intended to
be a user-facing interface.

[dnj, 2016/09/30 22:54:54]

Done.

+    # Get the package base for this OS.
+    _, base, dirname = self._package_spec(plat, arch)
+    name = '%s%s.zip' % (base, version)
+    artifact_url = '%s/%s' % (self.GS_BUCKET_BASE, name)
+
+    tdir = self.m.path.mkdtemp('gae_sdk')
+    dst = tdir.join(name) # Store the ZIP file here.
+    unzip_dir = tdir.join('unpack') # Unzip contents here.
+    self.m.gsutil.download_url(
+        artifact_url,
+        dst,
+        name=self._step_name('Download %s %s' % (plat, arch,)))
+    self.m.zip.unzip(
+        self._step_name('Unzip %s %s' % (plat, arch)),
+        dst,
+        unzip_dir,
+        quiet=True)
+
+    pkg_dir = unzip_dir.join(dirname)
+    self.m.path.mock_add_paths(pkg_dir)
+    assert self.m.path.exists(pkg_dir), (
+        'Package directory [%s] does not exist' % (pkg_dir,))
+    return pkg_dir
+
+  def fetch(self, plat, dst):

[iannucci, 2016/09/30 21:05:21]

it should somehow be made clear that this is really intended to be the ONLY
relevant function for users in this module. All the rest is really private to
the recipe (I would probably recommend putting that logic in the recipe until we
have multiple recipes that actually need it).

[dnj, 2016/09/30 22:54:54]

Done.

+    pkg = self.package(plat)
+    self.m.cipd.ensure(dst, {pkg: self.latest_ref})