Add CIPD pin reporting to swarming.

client/cipd.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Fetches CIPD client and installs packages."""
 
-__version__ = '0.2'
+__version__ = '0.3'
 
 import collections
 import contextlib
 import hashlib
+import json
 import logging
 import optparse
 import os
 import platform
 import sys
 import tempfile
 import time
 import urllib
 
 from utils import file_path
@@ skipping 75 matching lines @@
     parser.error(
         '--cipd-package requires non-empty --cipd-client-package')
   if not options.cipd_client_version:
     parser.error(
         '--cipd-package requires non-empty --cipd-client-version')
 
 
 class CipdClient(object):
   """Installs packages."""
 
-  def __init__(self, binary_path, service_url=None):
+  def __init__(self, binary_path, package_name, instance_id, service_url):
     """Initializes CipdClient.
 
     Args:
       binary_path (str): path to the CIPD client binary.
+      package_name (str): the CIPD package name for the client itself.
+      instance_id (str): the CIPD instance_id for the client itself.
       service_url (str): if not None, URL of the CIPD backend that overrides
         the default one.
     """
     self.binary_path = binary_path
+    self.package_name = package_name
+    self.instance_id = instance_id
     self.service_url = service_url
 
   def ensure(
       self, site_root, packages, cache_dir=None, tmp_dir=None, timeout=None):
     """Ensures that packages installed in |site_root| equals |packages| set.
 
     Blocking call.
 
     Args:
       site_root (str): where to install packages.
       packages: list of (package_template, version) tuples.
       cache_dir (str): if set, cache dir for cipd binary own cache.
         Typically contains packages and tags.
       tmp_dir (str): if not None, dir for temp files.
       timeout (int): if not None, timeout in seconds for this function to run.
 
+    Returns:
+      Pinned packages in the form of [(package_name, package_id)], which
+      correspond 1:1 with the input packages argument.
+
     Raises:
       Error if could not install packages or timed out.
     """
     timeoutfn = tools.sliding_timeout(timeout)
     logging.info('Installing packages %r into %s', packages, site_root)
 
     list_file_handle, list_file_path = tempfile.mkstemp(
         dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt')
+    json_out_file_handle, json_file_path = tempfile.mkstemp(
+      dir=tmp_dir, prefix=u'cipd-ensure-result-', suffix='.json')
+    os.close(json_out_file_handle)
+
     try:
       try:
         for pkg, version in packages:
           pkg = render_package_name_template(pkg)
           os.write(list_file_handle, '%s %s\n' % (pkg, version))
       finally:
         os.close(list_file_handle)
 
       cmd = [
         self.binary_path, 'ensure',
         '-root', site_root,
         '-list', list_file_path,
         '-verbose',  # this is safe because cipd-ensure does not print a lot
+        '-json-output', json_file_path,
       ]
       if cache_dir:
         cmd += ['-cache-dir', cache_dir]
       if self.service_url:
         cmd += ['-service-url', self.service_url]
 
       logging.debug('Running %r', cmd)
       process = subprocess42.Popen(
           cmd, stdout=subprocess42.PIPE, stderr=subprocess42.PIPE)
       output = []
@@ skipping 10 matching lines @@
         if pipe_name == 'stderr':
           logging.debug('cipd client: %s', line)
         else:
           logging.info('cipd client: %s', line)
 
       exit_code = process.wait(timeout=timeoutfn())
       if exit_code != 0:
         raise Error(
             'Could not install packages; exit code %d\noutput:%s' % (
             exit_code, '\n'.join(output)))
+      with open(json_file_path) as jfile:
+        result_json = json.load(jfile)
+      return [(x['package'], x['instance_id']) for x in result_json['result']]
     finally:
       fs.remove(list_file_path)
+      fs.remove(json_file_path)
 
 
 def get_platform():
   """Returns ${platform} parameter value.
 
   Borrowed from
   https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204
   """
   # linux, mac or windows.
   platform_variant = {
@@ skipping 197 matching lines @@
     # A single host can run multiple swarming bots, but ATM they do not share
     # same root bot directory. Thus, it is safe to use the same name for the
     # binary.
     binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX))
     if fs.isfile(binary_path):
       file_path.remove(binary_path)
 
     with instance_cache.getfileobj(instance_id) as f:
       isolateserver.putfile(f, binary_path, 0511)  # -r-x--x--x
 
-    yield CipdClient(binary_path)
+    yield CipdClient(binary_path, package_name=package_name,
+                     instance_id=instance_id, service_url=service_url)
 
 
 def parse_package_args(packages):
   """Parses --cipd-package arguments.
 
   Assumes |packages| were validated by validate_cipd_options.
 
   Returns:
-    A map {path: [(package, version)]}.
+    A list of [(path, package_name, version), ...]
   """
-  result = collections.defaultdict(list)
+  result = []
   for pkg in packages:
     path, name, version = pkg.split(':', 2)
-    path = path.replace('/', os.path.sep)
     if not name:
       raise Error('Invalid package "%s": package name is not specified' % pkg)
     if not version:
       raise Error('Invalid package "%s": version is not specified' % pkg)
-    result[path].append((name, version))
+    result.append((path, name, version))
   return result